From a33a0a614ffbf8ecd670150ee2138ba8a762f4d3 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Tue, 16 Nov 2021 18:15:59 +0100 Subject: find users for background scan one by one Signed-off-by: Robin Appelman --- apps/files/lib/BackgroundJob/ScanFiles.php | 42 ++++++++++++++---------------- 1 file changed, 20 insertions(+), 22 deletions(-) (limited to 'apps/files') diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php index f0c318882dd..37416aed521 100644 --- a/apps/files/lib/BackgroundJob/ScanFiles.php +++ b/apps/files/lib/BackgroundJob/ScanFiles.php @@ -21,6 +21,7 @@ * along with this program. If not, see * */ + namespace OCA\Files\BackgroundJob; use OC\Files\Utils\Scanner; @@ -40,8 +41,6 @@ use OCP\IUserManager; class ScanFiles extends \OC\BackgroundJob\TimedJob { /** @var IConfig */ private $config; - /** @var IUserManager */ - private $userManager; /** @var IEventDispatcher */ private $dispatcher; /** @var ILogger */ @@ -53,23 +52,20 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { /** * @param IConfig $config - * @param IUserManager $userManager * @param IEventDispatcher $dispatcher * @param ILogger $logger * @param IDBConnection $connection */ public function __construct( - IConfig $config, - IUserManager $userManager, + IConfig $config, IEventDispatcher $dispatcher, - ILogger $logger, - IDBConnection $connection + ILogger $logger, + IDBConnection $connection ) { // Run once per 10 minutes $this->setInterval(60 * 10); $this->config = $config; - $this->userManager = $userManager; $this->dispatcher = $dispatcher; $this->logger = $logger; $this->connection = $connection; @@ -81,10 +77,10 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { protected function runScanner(string $user) { try { $scanner = new Scanner( - $user, - null, - $this->dispatcher, - $this->logger + $user, + null, + $this->dispatcher, + $this->logger ); $scanner->backgroundScan(''); } catch (\Exception $e) { @@ -94,20 +90,20 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { } /** - * Find all storages which have unindexed files and return a user for each + * Find a storage which have unindexed files and return a user with access to the storage * - * @return string[] + * @return string|false */ - private function getUsersToScan(): array { + private function getUserToScan() { $query = $this->connection->getQueryBuilder(); - $query->select($query->func()->max('user_id')) + $query->select('user_id') ->from('filecache', 'f') ->innerJoin('f', 'mounts', 'm', $query->expr()->eq('storage_id', 'storage')) ->where($query->expr()->lt('size', $query->createNamedParameter(0, IQueryBuilder::PARAM_INT))) - ->groupBy('storage_id') - ->setMaxResults(self::USERS_PER_SESSION); + ->andWhere($query->expr()->gt('parent', $query->createNamedParameter(-1, IQueryBuilder::PARAM_INT))) + ->setMaxResults(1); - return $query->execute()->fetchAll(\PDO::FETCH_COLUMN); + return $query->execute()->fetchOne(); } /** @@ -119,10 +115,12 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { return; } - $users = $this->getUsersToScan(); - - foreach ($users as $user) { + $usersScanned = 0; + $user = $this->getUserToScan(); + while ($user && $usersScanned < self::USERS_PER_SESSION) { $this->runScanner($user); + $user = $this->getUserToScan(); + $usersScanned += 1; } } } -- cgit v1.2.3 From aa6ed02587df06d6fc9ff3534a8f0622d7bde47e Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 17 Nov 2021 15:04:51 +0100 Subject: stop background scan early if a users still has unscanned files after background scan Signed-off-by: Robin Appelman --- apps/files/lib/BackgroundJob/ScanFiles.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'apps/files') diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php index 37416aed521..250338e1262 100644 --- a/apps/files/lib/BackgroundJob/ScanFiles.php +++ b/apps/files/lib/BackgroundJob/ScanFiles.php @@ -116,11 +116,17 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { } $usersScanned = 0; + $lastUser = ''; $user = $this->getUserToScan(); - while ($user && $usersScanned < self::USERS_PER_SESSION) { + while ($user && $usersScanned < self::USERS_PER_SESSION && $lastUser !== $user) { $this->runScanner($user); + $lastUser = $user; $user = $this->getUserToScan(); $usersScanned += 1; } + + if ($lastUser === $user) { + $this->logger->warning("User $user still has unscanned files after running background scan, background scan might be stopped prematurely"); + } } } -- cgit v1.2.3 From dce2198f897df706df38ce8c40202b2b63110703 Mon Sep 17 00:00:00 2001 From: Robin Appelman Date: Wed, 17 Nov 2021 16:44:11 +0100 Subject: fix tests Signed-off-by: Robin Appelman --- apps/files/lib/BackgroundJob/ScanFiles.php | 7 +++---- apps/files/tests/BackgroundJob/ScanFilesTest.php | 3 --- 2 files changed, 3 insertions(+), 7 deletions(-) (limited to 'apps/files') diff --git a/apps/files/lib/BackgroundJob/ScanFiles.php b/apps/files/lib/BackgroundJob/ScanFiles.php index 250338e1262..84846b9b55d 100644 --- a/apps/files/lib/BackgroundJob/ScanFiles.php +++ b/apps/files/lib/BackgroundJob/ScanFiles.php @@ -30,7 +30,6 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\IConfig; use OCP\IDBConnection; use OCP\ILogger; -use OCP\IUserManager; /** * Class ScanFiles is a background job used to run the file scanner over the user @@ -57,10 +56,10 @@ class ScanFiles extends \OC\BackgroundJob\TimedJob { * @param IDBConnection $connection */ public function __construct( - IConfig $config, + IConfig $config, IEventDispatcher $dispatcher, - ILogger $logger, - IDBConnection $connection + ILogger $logger, + IDBConnection $connection ) { // Run once per 10 minutes $this->setInterval(60 * 10); diff --git a/apps/files/tests/BackgroundJob/ScanFilesTest.php b/apps/files/tests/BackgroundJob/ScanFilesTest.php index f1fb505fe53..04eeff4a30b 100644 --- a/apps/files/tests/BackgroundJob/ScanFilesTest.php +++ b/apps/files/tests/BackgroundJob/ScanFilesTest.php @@ -30,7 +30,6 @@ use OCP\EventDispatcher\IEventDispatcher; use OCP\IConfig; use OCP\ILogger; use OCP\IUser; -use OCP\IUserManager; use Test\TestCase; use Test\Traits\MountProviderTrait; use Test\Traits\UserTrait; @@ -54,7 +53,6 @@ class ScanFilesTest extends TestCase { parent::setUp(); $config = $this->createMock(IConfig::class); - $userManager = $this->createMock(IUserManager::class); $dispatcher = $this->createMock(IEventDispatcher::class); $logger = $this->createMock(ILogger::class); $connection = \OC::$server->getDatabaseConnection(); @@ -63,7 +61,6 @@ class ScanFilesTest extends TestCase { $this->scanFiles = $this->getMockBuilder('\OCA\Files\BackgroundJob\ScanFiles') ->setConstructorArgs([ $config, - $userManager, $dispatcher, $logger, $connection, -- cgit v1.2.3