Merge pull request #26322 from nextcloud/recent-search

Use file search for recent files and optimize getting share types

3 files changed, 98 insertions, 194 deletions

diff --git a/apps/files/lib/Controller/ApiController.php b/apps/files/lib/Controller/ApiController.php
index 0cf261af726..20f60cde30d 100644
--- a/apps/files/lib/Controller/ApiController.php
+++ b/apps/files/lib/Controller/ApiController.php
@@ -177,9 +177,9 @@ class ApiController extends Controller {
 	 * @return array
 	 */
 	private function formatNodes(array $nodes) {
-		return array_values(array_map(function (Node $node) {
-			/** @var \OC\Files\Node\Node $shareTypes */
-			$shareTypes = $this->getShareTypes($node);
+		$shareTypesForNodes = $this->getShareTypesForNodes($nodes);
+		return array_values(array_map(function (Node $node) use ($shareTypesForNodes) {
+			$shareTypes = $shareTypesForNodes[$node->getId()] ?? [];
 			$file = \OCA\Files\Helper::formatFileInfo($node->getFileInfo());
 			$file['hasPreview'] = $this->previewManager->isAvailable($node);
 			$parts = explode('/', dirname($node->getPath()), 4);
@@ -196,28 +196,13 @@ class ApiController extends Controller {
 	}
 
 	/**
-	 * Returns a list of recently modifed files.
-	 *
-	 * @NoAdminRequired
+	 * Get the share types for each node
 	 *
-	 * @return DataResponse
-	 */
-	public function getRecentFiles() {
-		$nodes = $this->userFolder->getRecent(100);
-		$files = $this->formatNodes($nodes);
-		return new DataResponse(['files' => $files]);
-	}
-
-	/**
-	 * Return a list of share types for outgoing shares
-	 *
-	 * @param Node $node file node
-	 *
-	 * @return int[] array of share types
+	 * @param \OCP\Files\Node[] $nodes
+	 * @return array<int, int[]> list of share types for each fileid
 	 */
-	private function getShareTypes(Node $node) {
+	private function getShareTypesForNodes(array $nodes): array {
 		$userId = $this->userSession->getUser()->getUID();
-		$shareTypes = [];
 		$requestedShareTypes = [
 			IShare::TYPE_USER,
 			IShare::TYPE_GROUP,
@@ -227,23 +212,54 @@ class ApiController extends Controller {
 			IShare::TYPE_ROOM,
 			IShare::TYPE_DECK,
 		];
-		foreach ($requestedShareTypes as $requestedShareType) {
-			// one of each type is enough to find out about the types
-			$shares = $this->shareManager->getSharesBy(
-				$userId,
-				$requestedShareType,
-				$node,
-				false,
-				1
-			);
-			if (!empty($shares)) {
-				$shareTypes[] = $requestedShareType;
+		$shareTypes = [];
+
+		$nodeIds = array_map(function (Node $node) {
+			return $node->getId();
+		}, $nodes);
+
+		foreach ($requestedShareTypes as $shareType) {
+			$nodesLeft = array_combine($nodeIds, array_fill(0, count($nodeIds), true));
+			$offset = 0;
+
+			// fetch shares until we've either found shares for all nodes or there are no more shares left
+			while (count($nodesLeft) > 0) {
+				$shares = $this->shareManager->getSharesBy($userId, $shareType, null, false, 100, $offset);
+				foreach ($shares as $share) {
+					$fileId = $share->getNodeId();
+					if (isset($nodesLeft[$fileId])) {
+						if (!isset($shareTypes[$fileId])) {
+							$shareTypes[$fileId] = [];
+						}
+						$shareTypes[$fileId][] = $shareType;
+						unset($nodesLeft[$fileId]);
+					}
+				}
+
+				if (count($shares) < 100) {
+					break;
+				} else {
+					$offset += count($shares);
+				}
 			}
 		}
 		return $shareTypes;
 	}
 
 	/**
+	 * Returns a list of recently modifed files.
+	 *
+	 * @NoAdminRequired
+	 *
+	 * @return DataResponse
+	 */
+	public function getRecentFiles() {
+		$nodes = $this->userFolder->getRecent(100);
+		$files = $this->formatNodes($nodes);
+		return new DataResponse(['files' => $files]);
+	}
+
+	/**
 	 * Change the default sort mode
 	 *
 	 * @NoAdminRequired
diff --git a/lib/private/Files/Node/Folder.php b/lib/private/Files/Node/Folder.php
index 14b663d4e16..f77c90b65a9 100644
--- a/lib/private/Files/Node/Folder.php
+++ b/lib/private/Files/Node/Folder.php
@@ -31,14 +31,10 @@
 
 namespace OC\Files\Node;
 
-use OC\DB\QueryBuilder\Literal;
 use OC\Files\Search\SearchBinaryOperator;
 use OC\Files\Search\SearchComparison;
+use OC\Files\Search\SearchOrder;
 use OC\Files\Search\SearchQuery;
-use OC\Files\Storage\Wrapper\Jail;
-use OC\Files\Storage\Storage;
-use OCA\Files_Sharing\SharedStorage;
-use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\ICacheEntry;
 use OCP\Files\Config\ICachedMountInfo;
 use OCP\Files\FileInfo;
@@ -48,6 +44,7 @@ use OCP\Files\NotPermittedException;
 use OCP\Files\Search\ISearchBinaryOperator;
 use OCP\Files\Search\ISearchComparison;
 use OCP\Files\Search\ISearchOperator;
+use OCP\Files\Search\ISearchOrder;
 use OCP\Files\Search\ISearchQuery;
 use OCP\IUserManager;
 
@@ -266,8 +263,7 @@ class Folder extends Node implements \OCP\Files\Folder {
 			new SearchBinaryOperator(ISearchBinaryOperator::OPERATOR_AND, [
 				new SearchComparison(ISearchComparison::COMPARE_LIKE, 'path', $internalPath . '%'),
 				$query->getSearchOperation(),
-			]
-			),
+			]),
 			$subQueryLimit,
 			0,
 			$query->getOrder(),
@@ -309,7 +305,7 @@ class Folder extends Node implements \OCP\Files\Folder {
 
 		$order = $query->getOrder();
 		if ($order) {
-			usort($files, function (FileInfo $a,FileInfo  $b) use ($order) {
+			usort($files, function (FileInfo $a, FileInfo $b) use ($order) {
 				foreach ($order as $orderField) {
 					$cmp = $orderField->sortFileInfo($a, $b);
 					if ($cmp !== 0) {
@@ -492,158 +488,37 @@ class Folder extends Node implements \OCP\Files\Folder {
 	 * @return \OCP\Files\Node[]
 	 */
 	public function getRecent($limit, $offset = 0) {
-		$mimetypeLoader = \OC::$server->getMimeTypeLoader();
-		$mounts = $this->root->getMountsIn($this->path);
-		$mounts[] = $this->getMountPoint();
-
-		$mounts = array_filter($mounts, function (IMountPoint $mount) {
-			return $mount->getStorage() !== null;
-		});
-		$storageIds = array_map(function (IMountPoint $mount) {
-			return $mount->getStorage()->getCache()->getNumericStorageId();
-		}, $mounts);
-		/** @var IMountPoint[] $mountMap */
-		$mountMap = array_combine($storageIds, $mounts);
-		$folderMimetype = $mimetypeLoader->getId(FileInfo::MIMETYPE_FOLDER);
-
-		/*
-		 * Construct an array of the storage id with their prefix path
-		 * This helps us to filter in the final query
-		 */
-		$filters = array_map(function (IMountPoint $mount) {
-			$storage = $mount->getStorage();
-
-			$storageId = $storage->getCache()->getNumericStorageId();
-			$prefix = '';
-
-			if ($storage->instanceOfStorage(Jail::class)) {
-				$prefix = $storage->getUnJailedPath('');
-			}
-
-			return [
-				'storageId' => $storageId,
-				'pathPrefix' => $prefix,
-			];
-		}, $mounts);
-
-		// Search in batches of 500 entries
-		$searchLimit = 500;
-		$results = [];
-		$searchResultCount = 0;
-		$count = 0;
-		do {
-			$searchResult = $this->recentSearch($searchLimit, $offset, $folderMimetype, $filters);
-
-			// Exit condition if there are no more results
-			if (count($searchResult) === 0) {
-				break;
-			}
-
-			$searchResultCount += count($searchResult);
-
-			$parseResult = $this->recentParse($searchResult, $mountMap, $mimetypeLoader);
-
-			foreach ($parseResult as $result) {
-				$results[] = $result;
-			}
-
-			$offset += $searchLimit;
-			$count++;
-		} while (count($results) < $limit && ($searchResultCount < (3 * $limit) || $count < 5));
-
-		return array_slice($results, 0, $limit);
-	}
-
-	private function recentSearch($limit, $offset, $folderMimetype, $filters) {
-		$dbconn = \OC::$server->getDatabaseConnection();
-		$builder = $dbconn->getQueryBuilder();
-		$query = $builder
-			->select('f.*')
-			->from('filecache', 'f');
-
-		/*
-		 * Here is where we construct the filtering.
-		 * Note that this is expensive filtering as it is a lot of like queries.
-		 * However the alternative is we do this filtering and parsing later in php with the risk of looping endlessly
-		 */
-		$storageFilters = $builder->expr()->orX();
-		foreach ($filters as $filter) {
-			$storageFilter = $builder->expr()->andX(
-				$builder->expr()->eq('f.storage', $builder->createNamedParameter($filter['storageId']))
-			);
-
-			if ($filter['pathPrefix'] !== '') {
-				$storageFilter->add(
-					$builder->expr()->like('f.path', $builder->createNamedParameter($dbconn->escapeLikeParameter($filter['pathPrefix']) . '/%'))
-				);
-			}
-
-			$storageFilters->add($storageFilter);
-		}
-
-		$query->andWhere($storageFilters);
-
-		$query->andWhere($builder->expr()->orX(
-		// handle non empty folders separate
-			$builder->expr()->neq('f.mimetype', $builder->createNamedParameter($folderMimetype, IQueryBuilder::PARAM_INT)),
-			$builder->expr()->eq('f.size', new Literal(0))
-		))
-			->andWhere($builder->expr()->notLike('f.path', $builder->createNamedParameter('files_versions/%')))
-			->andWhere($builder->expr()->notLike('f.path', $builder->createNamedParameter('files_trashbin/%')))
-			->orderBy('f.mtime', 'DESC')
-			->setMaxResults($limit)
-			->setFirstResult($offset);
-
-		$result = $query->execute();
-		$rows = $result->fetchAll();
-		$result->closeCursor();
-
-		return $rows;
-	}
-
-	private function recentParse($result, $mountMap, $mimetypeLoader) {
-		$files = array_filter(array_map(function (array $entry) use ($mountMap, $mimetypeLoader) {
-			$mount = $mountMap[$entry['storage']];
-			$entry['internalPath'] = $entry['path'];
-			$entry['mimetype'] = $mimetypeLoader->getMimetypeById($entry['mimetype']);
-			$entry['mimepart'] = $mimetypeLoader->getMimetypeById($entry['mimepart']);
-			$path = $this->getAbsolutePath($mount, $entry['path']);
-			if (is_null($path)) {
-				return null;
-			}
-			$fileInfo = new \OC\Files\FileInfo($path, $mount->getStorage(), $entry['internalPath'], $entry, $mount);
-			return $this->root->createNode($fileInfo->getPath(), $fileInfo);
-		}, $result));
-
-		return array_values(array_filter($files, function (Node $node) {
-			$cacheEntry = $node->getMountPoint()->getStorage()->getCache()->get($node->getId());
-			if (!$cacheEntry) {
-				return false;
-			}
-			$relative = $this->getRelativePath($node->getPath());
-			return $relative !== null && $relative !== '/'
-				&& ($cacheEntry->getPermissions() & \OCP\Constants::PERMISSION_READ) === \OCP\Constants::PERMISSION_READ;
-		}));
-	}
-
-	private function getAbsolutePath(IMountPoint $mount, $path) {
-		$storage = $mount->getStorage();
-		if ($storage->instanceOfStorage('\OC\Files\Storage\Wrapper\Jail')) {
-			if ($storage->instanceOfStorage(SharedStorage::class)) {
-				$storage->getSourceStorage();
-			}
-			/** @var \OC\Files\Storage\Wrapper\Jail $storage */
-			$jailRoot = $storage->getUnjailedPath('');
-			$rootLength = strlen($jailRoot) + 1;
-			if ($path === $jailRoot) {
-				return $mount->getMountPoint();
-			} elseif (substr($path, 0, $rootLength) === $jailRoot . '/') {
-				return $mount->getMountPoint() . substr($path, $rootLength);
-			} else {
-				return null;
-			}
-		} else {
-			return $mount->getMountPoint() . $path;
-		}
+		$query = new SearchQuery(
+			new SearchBinaryOperator(
+				// filter out non empty folders
+				ISearchBinaryOperator::OPERATOR_OR,
+				[
+					new SearchBinaryOperator(
+						ISearchBinaryOperator::OPERATOR_NOT,
+						[
+							new SearchComparison(
+								ISearchComparison::COMPARE_EQUAL,
+								'mimetype',
+								FileInfo::MIMETYPE_FOLDER
+							),
+						]
+					),
+					new SearchComparison(
+						ISearchComparison::COMPARE_EQUAL,
+						'size',
+						0
+					),
+				]
+			),
+			$limit,
+			$offset,
+			[
+				new SearchOrder(
+					ISearchOrder::DIRECTION_DESCENDING,
+					'mtime'
+				),
+			]
+		);
+		return $this->search($query);
 	}
 }
diff --git a/tests/lib/Files/Node/FolderTest.php b/tests/lib/Files/Node/FolderTest.php
index 1d541556c0b..36398171b1b 100644
--- a/tests/lib/Files/Node/FolderTest.php
+++ b/tests/lib/Files/Node/FolderTest.php
@@ -773,6 +773,10 @@ class FolderTest extends NodeTest {
 		$folderInfo->expects($this->any())
 			->method('getMountPoint')
 			->willReturn($mount);
+		$root->method('getMount')
+			->willReturn($mount);
+		$root->method('getMountsIn')
+			->willReturn([]);
 
 		$cache = $storage->getCache();
 
@@ -838,6 +842,11 @@ class FolderTest extends NodeTest {
 			->method('getMountPoint')
 			->willReturn($mount);
 
+		$root->method('getMount')
+			->willReturn($mount);
+		$root->method('getMountsIn')
+			->willReturn([]);
+
 		$cache = $storage->getCache();
 
 		$id1 = $cache->put('bar/foo/folder', [
@@ -903,6 +912,10 @@ class FolderTest extends NodeTest {
 		$folderInfo->expects($this->any())
 			->method('getMountPoint')
 			->willReturn($mount);
+		$root->method('getMount')
+			->willReturn($mount);
+		$root->method('getMountsIn')
+			->willReturn([]);
 
 		$cache = $storage->getCache();