From 96f407663de728cbe80954d8a09ab16c26ef4790 Mon Sep 17 00:00:00 2001 From: Maxence Lange Date: Wed, 4 May 2022 20:03:39 -0100 Subject: collections and external API Signed-off-by: Maxence Lange Co-authored-by: Carl Schwan --- appinfo/info.xml | 5 + appinfo/routes.php | 15 ++ lib/Command/CollectionDelete.php | 89 ++++++++ lib/Command/CollectionInit.php | 198 ++++++++++++++++ lib/Command/CollectionList.php | 83 +++++++ lib/Command/Index.php | 110 ++++----- lib/Command/Migration24.php | 135 +++++++++++ lib/Command/Reset.php | 23 +- lib/Controller/CollectionController.php | 164 ++++++++++++++ lib/Cron/Index.php | 2 +- lib/Cron/Maintenance.php | 74 ++++++ lib/Db/CoreRequestBuilder.php | 13 +- lib/Db/IndexesRequest.php | 159 +++++++++---- lib/Db/IndexesRequestBuilder.php | 7 +- lib/Exceptions/CollectionArgumentException.php | 40 ++++ lib/Migration/Version2000Date20201208130255.php | 40 ---- lib/Migration/Version2400Date202201301329.php | 162 ++++++++++++++ lib/Model/Index.php | 62 +++++- lib/Model/Runner.php | 29 ++- lib/Service/CliService.php | 4 +- lib/Service/CollectionService.php | 285 ++++++++++++++++++++++++ lib/Service/ConfigService.php | 42 +++- lib/Service/IndexService.php | 204 ++++++++--------- lib/Service/MigrationService.php | 182 +++++++++++++++ lib/Service/ProviderService.php | 13 +- 25 files changed, 1850 insertions(+), 290 deletions(-) create mode 100644 lib/Command/CollectionDelete.php create mode 100644 lib/Command/CollectionInit.php create mode 100644 lib/Command/CollectionList.php create mode 100644 lib/Command/Migration24.php create mode 100644 lib/Controller/CollectionController.php create mode 100644 lib/Cron/Maintenance.php create mode 100644 lib/Exceptions/CollectionArgumentException.php create mode 100644 lib/Migration/Version2400Date202201301329.php create mode 100644 lib/Service/CollectionService.php create mode 100644 lib/Service/MigrationService.php diff --git a/appinfo/info.xml b/appinfo/info.xml index 16ae8e5..4dbd150 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -28,10 +28,14 @@ Core App of the full-text search framework for your Nextcloud. OCA\FullTextSearch\Cron\Index + OCA\FullTextSearch\Cron\Maintenance OCA\FullTextSearch\Command\Check + OCA\FullTextSearch\Command\CollectionInit + OCA\FullTextSearch\Command\CollectionDelete + OCA\FullTextSearch\Command\CollectionList OCA\FullTextSearch\Command\Configure OCA\FullTextSearch\Command\DocumentIndex OCA\FullTextSearch\Command\DocumentPlatform @@ -39,6 +43,7 @@ Core App of the full-text search framework for your Nextcloud. OCA\FullTextSearch\Command\DocumentStatus OCA\FullTextSearch\Command\Index OCA\FullTextSearch\Command\Live + OCA\FullTextSearch\Command\Migration24 OCA\FullTextSearch\Command\Reset OCA\FullTextSearch\Command\Search OCA\FullTextSearch\Command\Stop diff --git a/appinfo/routes.php b/appinfo/routes.php index 967ac82..5b56710 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -29,6 +29,21 @@ declare(strict_types=1); return [ + 'ocs' => [ + /** @see OCA\FullTextSearch\Controller\CollectionController */ + ['name' => 'Collection#getQueue', 'url' => '/collection/{collection}/index', 'verb' => 'GET'], + [ + 'name' => 'Collection#indexDocument', + 'url' => '/collection/{collection}/document/{providerId}/{documentId}', + 'verb' => 'GET' + ], + [ + 'name' => 'Collection#updateStatusDone', + 'url' => '/collection/{collection}/document/{providerId}/{documentId}/done', + 'verb' => 'POST' + ] + ], + 'routes' => [ ['name' => 'Navigation#navigate', 'url' => '/', 'verb' => 'GET'], ['name' => 'Settings#getSettingsAdmin', 'url' => '/admin/settings', 'verb' => 'GET'], diff --git a/lib/Command/CollectionDelete.php b/lib/Command/CollectionDelete.php new file mode 100644 index 0000000..98b0a23 --- /dev/null +++ b/lib/Command/CollectionDelete.php @@ -0,0 +1,89 @@ + + * @copyright 2018 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Command; + + +use OC\Core\Command\Base; +use OCA\FullTextSearch\Exceptions\CollectionArgumentException; +use OCA\FullTextSearch\Service\CollectionService; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class CollectionDelete extends Base { + + + /** @var CollectionService */ + private $collectionService; + + + /** + * @param CollectionService $collectionService + */ + public function __construct(CollectionService $collectionService) { + parent::__construct(); + + $this->collectionService = $collectionService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('fulltextsearch:collection:delete') + ->setDescription('Delete collection') + ->addArgument('name', InputArgument::REQUIRED, 'name of the collection to delete'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $collection = $input->getArgument('name'); + if (!$this->collectionService->hasCollection($collection)) { + throw new CollectionArgumentException('unknown collection'); + } + + $this->collectionService->deleteCollection($collection); + + return 0; + } +} + + + diff --git a/lib/Command/CollectionInit.php b/lib/Command/CollectionInit.php new file mode 100644 index 0000000..3f5c263 --- /dev/null +++ b/lib/Command/CollectionInit.php @@ -0,0 +1,198 @@ + + * @copyright 2018 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Command; + + +use Exception; +use OC\Core\Command\Base; +use OCA\FullTextSearch\Model\IndexOptions; +use OCA\FullTextSearch\Model\Runner; +use OCA\FullTextSearch\Service\CliService; +use OCA\FullTextSearch\Service\CollectionService; +use OCA\FullTextSearch\Service\ProviderService; +use OCA\FullTextSearch\Service\RunningService; +use OCP\FullTextSearch\IFullTextSearchProvider; +use OCP\IUserManager; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class CollectionInit extends Base { + + + /** @var ProviderService */ + private $providerService; + + /** @var CollectionService */ + private $collectionService; + + /** @var IUserManager */ + private $userManager; + + /** @var RunningService */ + private $runningService; + + /** @var CliService */ + private $cliService; + + + /** + * @param IUserManager $userManager + * @param CollectionService $collectionService + * @param ProviderService $providerService + * @param RunningService $runningService + * @param CliService $cliService + */ + public function __construct( + IUserManager $userManager, + CollectionService $collectionService, + ProviderService $providerService, + RunningService $runningService, + CliService $cliService + ) { + parent::__construct(); + + $this->userManager = $userManager; + $this->collectionService = $collectionService; + $this->providerService = $providerService; + $this->runningService = $runningService; + $this->cliService = $cliService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('fulltextsearch:collection:init') + ->setDescription('Initiate a collection') + ->addArgument('name', InputArgument::REQUIRED, 'name of the collection'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + * @throws Exception + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $collection = $input->getArgument('name'); + $this->collectionService->confirmCollectionString($collection); + + $runner = new Runner($this->runningService, 'commandIndex', ['nextStep' => 'n']); +// $runner->sourceIsCommandLine($this, $output); + $this->collectionService->setRunner($runner); + $this->cliService->setRunner($runner); + + $this->cliService->createPanel( + 'collection', [ + '┌─ Collection ' . $collection . ' ────', + '│ ProviderId, UserId: %providerId% / %userId%', + '│ Chunk: %chunkCurr:3s%/%chunkTotal%', + '│ Document: %documentCurr:6s%/%documentChunk%', + '│', + '│ Total Document: %documentTotal%', + '│ Index initiated: %indexCount%', + '└──' + ] + ); + + $runner->setInfoArray([ + 'providerId' => '', + 'userId' => '', + 'chunkCurr' => '', + 'chunkTotal' => '', + 'documentCurr' => '', + 'documentChunk' => '', + 'documentTotal' => '', + 'indexCount' => 0 + ]); + + $this->cliService->initDisplay(); + $this->cliService->displayPanel('run', 'collection'); + $this->cliService->runDisplay($output); + + $providers = $this->providerService->getProviders(); + foreach ($providers as $providerWrapper) { + $this->indexProvider($runner, $collection, $providerWrapper->getProvider()); + } + + + return 0; + } + + + /** + * @param string $collection + * @param IFullTextSearchProvider $provider + * + * @throws Exception + */ + private function indexProvider( + Runner $runner, + string $collection, + IFullTextSearchProvider $provider, + string $userId = '' + ) { + $runner->setInfo('providerId', $provider->getId()); + $options = new IndexOptions(); + $provider->setIndexOptions($options); + + if ($userId === '') { + $users = $this->userManager->search(''); + } else { + $users = [$userId]; + } + + foreach ($users as $user) { + if ($user === null) { + continue; + } + + $runner->setInfo('userId', $user->getUID()); + + try { + $this->collectionService->initCollectionIndexes( + $provider, + $collection, + $user->getUID(), + $options + ); + } catch (Exception $e) { + continue; + } + } + } +} diff --git a/lib/Command/CollectionList.php b/lib/Command/CollectionList.php new file mode 100644 index 0000000..c267107 --- /dev/null +++ b/lib/Command/CollectionList.php @@ -0,0 +1,83 @@ + + * @copyright 2018 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Command; + + +use OC\Core\Command\Base; +use OCA\FullTextSearch\Service\CollectionService; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class CollectionList extends Base { + + + /** @var CollectionService */ + private $collectionService; + + + /** + * @param CollectionService $collectionService + */ + public function __construct(CollectionService $collectionService) { + parent::__construct(); + + $this->collectionService = $collectionService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('fulltextsearch:collection:list') + ->setDescription('List collections'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + */ + protected function execute(InputInterface $input, OutputInterface $output) { + $collections = $this->collectionService->getCollections(); + $output->writeln('found ' . sizeof($collections) . ' collection(s)'); + + foreach ($this->collectionService->getCollections() as $collection) { + $output->writeln('- ' . $collection); + } + + return 0; + } +} diff --git a/lib/Command/Index.php b/lib/Command/Index.php index 73211f2..1508769 100644 --- a/lib/Command/Index.php +++ b/lib/Command/Index.php @@ -40,6 +40,7 @@ use OCA\FullTextSearch\Model\Index as ModelIndex; use OCA\FullTextSearch\Model\IndexOptions; use OCA\FullTextSearch\Model\Runner; use OCA\FullTextSearch\Service\CliService; +use OCA\FullTextSearch\Service\ConfigService; use OCA\FullTextSearch\Service\IndexService; use OCA\FullTextSearch\Service\MiscService; use OCA\FullTextSearch\Service\PlatformService; @@ -48,7 +49,6 @@ use OCA\FullTextSearch\Service\RunningService; use OCP\FullTextSearch\IFullTextSearchProvider; use OCP\IUserManager; use OutOfBoundsException; -use Symfony\Component\Console\Formatter\OutputFormatterStyle; use Symfony\Component\Console\Input\InputArgument; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; @@ -138,6 +138,8 @@ class Index extends ACommandBase { /** @var MiscService */ private $miscService; + /** @var ConfigService */ + private $configService; /** @var Runner */ private $runner; @@ -168,11 +170,13 @@ class Index extends ACommandBase { * @param PlatformService $platformService * @param ProviderService $providerService * @param MiscService $miscService + * @param ConfigService $configService */ public function __construct( IUserManager $userManager, RunningService $runningService, CliService $cliService, IndexService $indexService, PlatformService $platformService, - ProviderService $providerService, MiscService $miscService + ProviderService $providerService, MiscService $miscService, + ConfigService $configService ) { parent::__construct(); $this->userManager = $userManager; @@ -184,6 +188,7 @@ class Index extends ACommandBase { $this->platformService = $platformService; $this->providerService = $providerService; $this->miscService = $miscService; + $this->configService = $configService; } @@ -210,6 +215,7 @@ class Index extends ACommandBase { * @throws Exception */ protected function execute(InputInterface $input, OutputInterface $output) { + $this->configService->requireMigration24(); $options = $this->generateIndexOptions($input); @@ -217,7 +223,7 @@ class Index extends ACommandBase { /** do not get stuck while waiting interactive input */ try { readline_callback_handler_install( - '', function() { + '', function () { } ); } catch (Throwable $t) { @@ -414,7 +420,7 @@ class Index extends ACommandBase { } } - $this->providerService->setProviderAsIndexed($provider, true); + $this->providerService->setProviderAsIndexed($provider->getId(), true); } @@ -572,37 +578,37 @@ class Index extends ACommandBase { // full list of info that can be edited $this->runner->setInfoArray( [ - 'userId' => '', + 'userId' => '', 'providerName' => '', - '_memory' => '', - 'documentId' => '', - 'action' => '', - 'info' => '', - 'title' => '', - '_paused' => '', - - 'resultIndex' => '', - 'resultCurrent' => '', - 'resultTotal' => '', - 'resultMessageA' => '', - 'resultMessageB' => '', - 'resultMessageC' => '', - 'resultStatus' => '', + '_memory' => '', + 'documentId' => '', + 'action' => '', + 'info' => '', + 'title' => '', + '_paused' => '', + + 'resultIndex' => '', + 'resultCurrent' => '', + 'resultTotal' => '', + 'resultMessageA' => '', + 'resultMessageB' => '', + 'resultMessageC' => '', + 'resultStatus' => '', 'resultStatusColored' => '', - 'content' => '', - 'statusColored' => '', - 'chunkCurrent' => '', - 'chunkTotal' => '', - 'documentCurrent' => '', - 'documentTotal' => '', - 'progressStatus' => '', - 'errorCurrent' => '0', - 'errorTotal' => '0', - 'errorMessageA' => '', - 'errorMessageB' => '', - 'errorMessageC' => '', - 'errorException' => '', - 'errorIndex' => '' + 'content' => '', + 'statusColored' => '', + 'chunkCurrent' => '', + 'chunkTotal' => '', + 'documentCurrent' => '', + 'documentTotal' => '', + 'progressStatus' => '', + 'errorCurrent' => '0', + 'errorTotal' => '0', + 'errorMessageA' => '', + 'errorMessageB' => '', + 'errorMessageC' => '', + 'errorException' => '', + 'errorIndex' => '' ] ); } @@ -618,7 +624,7 @@ class Index extends ACommandBase { $this->runner->setInfoArray( [ 'errorCurrent' => 0, - 'errorTotal' => 0, + 'errorTotal' => 0, ] ); @@ -648,13 +654,13 @@ class Index extends ACommandBase { $this->runner->setInfoArray( [ - 'errorCurrent' => $current, - 'errorTotal' => $total, - 'errorMessageA' => trim($err1), - 'errorMessageB' => trim($err2), - 'errorMessageC' => trim($err3), + 'errorCurrent' => $current, + 'errorTotal' => $total, + 'errorMessageA' => trim($err1), + 'errorMessageB' => trim($err2), + 'errorMessageC' => trim($err3), 'errorException' => $this->get('exception', $error, ''), - 'errorIndex' => $errorIndex + 'errorIndex' => $errorIndex ] ); } @@ -670,7 +676,7 @@ class Index extends ACommandBase { $this->runner->setInfoArray( [ 'resultCurrent' => 0, - 'resultTotal' => 0, + 'resultTotal' => 0, ] ); @@ -704,13 +710,13 @@ class Index extends ACommandBase { $this->runner->setInfoArray( [ - 'resultCurrent' => $current, - 'resultTotal' => $total, + 'resultCurrent' => $current, + 'resultTotal' => $total, 'resultMessageA' => trim($msg1), 'resultMessageB' => trim($msg2), 'resultMessageC' => trim($msg3), - 'resultStatus' => $status, - 'resultIndex' => $resultIndex + 'resultStatus' => $status, + 'resultIndex' => $resultIndex ] ); $this->runner->setInfoColored('resultStatus', $type); @@ -808,10 +814,10 @@ class Index extends ACommandBase { foreach ($indexes as $index) { foreach ($index->getErrors() as $error) { $this->errors[] = [ - 'index' => $index, - 'message' => $error['message'], + 'index' => $index, + 'message' => $error['message'], 'exception' => $error['exception'], - 'severity' => $error['severity'] + 'severity' => $error['severity'] ]; } @@ -830,11 +836,11 @@ class Index extends ACommandBase { $this->runner->setInfoArray( [ - 'errorMessageA' => '', - 'errorMessageB' => '', - 'errorMessageC' => '', + 'errorMessageA' => '', + 'errorMessageB' => '', + 'errorMessageC' => '', 'errorException' => '', - 'errorIndex' => '' + 'errorIndex' => '' ] ); diff --git a/lib/Command/Migration24.php b/lib/Command/Migration24.php new file mode 100644 index 0000000..0d9a899 --- /dev/null +++ b/lib/Command/Migration24.php @@ -0,0 +1,135 @@ + + * @copyright 2018 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Command; + + +use OC\Core\Command\Base; +use OCA\FullTextSearch\Exceptions\DatabaseException; +use OCA\FullTextSearch\Service\ConfigService; +use OCA\FullTextSearch\Service\MigrationService; +use OCP\DB\Exception; +use OCP\IDBConnection; +use Symfony\Component\Console\Helper\ProgressBar; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + + +class Migration24 extends Base { + + + /** @var IDBConnection */ + private $dbConnection; + + /** @var MigrationService */ + private $migrationService; + + /** @var ConfigService */ + private $configService; + + + /** + * @param IDBConnection $dbConnection + * @param MigrationService $migrationService + * @param ConfigService $configService + */ + public function __construct( + IDBConnection $dbConnection, + MigrationService $migrationService, + ConfigService $configService + ) { + parent::__construct(); + + $this->dbConnection = $dbConnection; + $this->migrationService = $migrationService; + $this->configService = $configService; + } + + + /** + * + */ + protected function configure() { + parent::configure(); + $this->setName('fulltextsearch:migration:24') + ->setDescription('Migrate index for NC24'); + } + + + /** + * @param InputInterface $input + * @param OutputInterface $output + * + * @return int + * @throws \Exception + */ + protected function execute(InputInterface $input, OutputInterface $output): int { + try { + $this->configService->requireMigration24(); + throw new \Exception('Should not be required'); + } catch (DatabaseException $e) { + } + + $this->runMigration($output); + + return 0; + } + + + private function runMigration(OutputInterface $output): void { + $progressBar = new ProgressBar($output, $this->getTotalRows()); + $this->migrationService->setProgressBar($progressBar); + while (true) { + if (!$this->migrationService->migrate24Chunk(10000, $progressBar)) { + break; + } + } + + $progressBar->finish(); + $output->writeln(''); + } + + + /** + * @return int + * @throws Exception + */ + private function getTotalRows(): int { + $query = $this->dbConnection->getQueryBuilder(); + $query->select($query->func()->count('*', 'index_count')) + ->from('fulltextsearch_indexes'); + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + return (int)$row['index_count']; + } + +} diff --git a/lib/Command/Reset.php b/lib/Command/Reset.php index c5aa80e..a1e80ab 100644 --- a/lib/Command/Reset.php +++ b/lib/Command/Reset.php @@ -87,7 +87,8 @@ class Reset extends ACommandBase { parent::configure(); $this->setName('fulltextsearch:reset') ->setDescription('Reset index') - ->addArgument('provider', InputArgument::OPTIONAL, 'provider'); + ->addArgument('provider', InputArgument::OPTIONAL, 'provider id', '') + ->addArgument('collection', InputArgument::OPTIONAL, 'name of the collection', ''); } @@ -111,7 +112,10 @@ class Reset extends ACommandBase { $this->indexService->setRunner($this->runner); try { - $this->indexService->resetIndex($this->getProviderIdFromArgument($input)); + $this->indexService->resetIndex( + $input->getArgument('provider'), + $input->getArgument('collection') + ); } catch (Exception $e) { throw $e; @@ -123,21 +127,6 @@ class Reset extends ACommandBase { } - /** - * @param InputInterface $input - * - * @return string - */ - private function getProviderIdFromArgument(InputInterface $input): string { - $providerId = $input->getArgument('provider'); - if ($providerId === null) { - $providerId = ''; - } - - return $providerId; - } - - /** * @throws TickDoesNotExistException */ diff --git a/lib/Controller/CollectionController.php b/lib/Controller/CollectionController.php new file mode 100644 index 0000000..26f7f1a --- /dev/null +++ b/lib/Controller/CollectionController.php @@ -0,0 +1,164 @@ + + * @copyright 2022 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Controller; + + +use OCA\FullTextSearch\AppInfo\Application; +use OCA\FullTextSearch\Service\CollectionService; +use OCP\AppFramework\Http\DataResponse; +use OCP\AppFramework\OCS\OCSException; +use OCP\AppFramework\OCSController; +use OCP\FullTextSearch\Model\IIndexDocument; +use OCP\IRequest; + + +class CollectionController extends OCSController { + + + /** @var CollectionService */ + private $collectionService; + + + /** + * @param IRequest $request + * @param CollectionService $collectionService + */ + public function __construct( + IRequest $request, + CollectionService $collectionService + ) { + parent::__construct(Application::APP_ID, $request); + + $this->collectionService = $collectionService; + } + + + /** + * @param string $collection + * @param int $length + * + * @return DataResponse + * @throws OCSException + */ + public function getQueue(string $collection, int $length = 0): DataResponse { + try { + $this->collectionService->confirmCollection($collection); + + return new DataResponse($this->collectionService->getQueue($collection, $length)); + } catch (\Exception $e) { +// $this->e($e, ['circleId' => $circleId]); + throw new OCSException($e->getMessage(), $e->getCode()); + } + } + + + /** + * @param string $collection + * @param string $providerId + * @param string $documentId + * + * @return DataResponse + * @throws OCSException + */ + public function indexDocument(string $collection, string $providerId, string $documentId): DataResponse { + try { + $this->collectionService->confirmCollection($collection); + $document = $this->collectionService->getDocument( + $collection, + $providerId, + $documentId + ); + + return new DataResponse($this->displayDocument($document)); + } catch (\Exception $e) { + throw new OCSException($e->getMessage(), $e->getCode()); + } + } + + + /** + * @param string $collection + * @param string $providerId + * @param string $documentId + * + * @return DataResponse + * @throws OCSException + */ + public function updateStatusDone( + string $collection, + string $providerId, + string $documentId + ): DataResponse { + try { + $this->collectionService->confirmCollection($collection); + $this->collectionService->setAsDone($collection, $providerId, $documentId); + + return new DataResponse([]); + } catch (\Exception $e) { + throw new OCSException($e->getMessage(), $e->getCode()); + } + } + + + /** + * @param IIndexDocument $document + * + * @return array + */ + private function displayDocument(IIndexDocument $document): array { + $display = [ + 'id' => $document->getId(), + 'providerId' => $document->getProviderId(), + 'access' => $document->getAccess(), + 'index' => $document->getIndex(), + 'title' => $document->getTitle(), + 'link' => $document->getLink(), + 'parts' => $document->getParts(), + 'tags' => $document->getTags(), + 'metatags' => $document->getMetaTags(), + 'source' => $document->getSource(), + 'info' => $document->getInfoAll(), + 'hash' => $document->getHash(), + 'modifiedTime' => $document->getModifiedTime() + ]; + + foreach ($document->getMore() as $k => $v) { + $display[$k] = $v; + } + + $display['content'] = $document->getContent(); + $display['isContentEncoded'] = $document->isContentEncoded(); + + return json_decode(json_encode($display), true); + } + +} + diff --git a/lib/Cron/Index.php b/lib/Cron/Index.php index 8374583..49374fe 100644 --- a/lib/Cron/Index.php +++ b/lib/Cron/Index.php @@ -119,7 +119,7 @@ class Index extends TimedJob { $platform = $wrapper->getPlatform(); $all = $this->shouldWeGetAllIndex(); - $indexes = $this->indexService->getQueuedIndexes($all); + $indexes = $this->indexService->getQueuedIndexes('', $all); foreach ($indexes as $index) { $this->runner->updateAction('indexing'); diff --git a/lib/Cron/Maintenance.php b/lib/Cron/Maintenance.php new file mode 100644 index 0000000..5a2d505 --- /dev/null +++ b/lib/Cron/Maintenance.php @@ -0,0 +1,74 @@ + + * @copyright 2022 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Cron; + + +use OC\BackgroundJob\TimedJob; +use OCA\FullTextSearch\Service\ConfigService; +use OCA\FullTextSearch\Service\MigrationService; + + +class Maintenance extends TimedJob { + + + /** @var MigrationService */ + private $migrationService; + + /** @var ConfigService */ + private $configService; + + + /** + * + */ + public function __construct(MigrationService $migrationService, ConfigService $configService) { + $this->setInterval(3600); + + $this->migrationService = $migrationService; + $this->configService = $configService; + } + + + /** + * @param mixed $argument + * + * @throws \OCP\DB\Exception + */ + protected function run($argument) { + $size = $this->configService->getAppValue('size_migration_24'); + if ($size === '') { + $size = 10000; + } + + $this->migrationService->migrate24Chunk((int)$size); + } + +} diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php index d90054c..47018cc 100644 --- a/lib/Db/CoreRequestBuilder.php +++ b/lib/Db/CoreRequestBuilder.php @@ -47,7 +47,7 @@ use OCP\IL10N; */ class CoreRequestBuilder { - const TABLE_INDEXES = 'fulltextsearch_indexes'; + const TABLE_INDEXES = 'fulltextsearch_index'; const TABLE_TICKS = 'fulltextsearch_ticks'; /** @var IDBConnection */ @@ -129,6 +129,17 @@ class CoreRequestBuilder { } + /** + * Limit to the documentId + * + * @param IQueryBuilder $qb + * @param string $collection + */ + protected function limitToCollection(IQueryBuilder &$qb, string $collection) { + $this->limitToDBField($qb, 'collection', $collection); + } + + /** * Limit to the entry with at least one Error * diff --git a/lib/Db/IndexesRequest.php b/lib/Db/IndexesRequest.php index f0c7ab9..99745d9 100644 --- a/lib/Db/IndexesRequest.php +++ b/lib/Db/IndexesRequest.php @@ -31,9 +31,11 @@ declare(strict_types=1); namespace OCA\FullTextSearch\Db; +use OCA\FullTextSearch\Exceptions\DatabaseException; use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException; use OCA\FullTextSearch\Model\Index; use OCP\FullTextSearch\Model\IIndex; +use OCP\IDBConnection; /** @@ -53,6 +55,7 @@ class IndexesRequest extends IndexesRequestBuilder { $qb = $this->getIndexesInsertSql(); $qb->setValue('owner_id', $qb->createNamedParameter($index->getOwnerId())) ->setValue('provider_id', $qb->createNamedParameter($index->getProviderId())) + ->setValue('collection', $qb->createNamedParameter($index->getCollection())) ->setValue('document_id', $qb->createNamedParameter($index->getDocumentId())) ->setValue('source', $qb->createNamedParameter($index->getSource())) ->setValue('err', $qb->createNamedParameter($index->getErrorCount())) @@ -73,9 +76,8 @@ class IndexesRequest extends IndexesRequestBuilder { * @return bool */ public function resetError(Index $index): bool { - try { - $this->getIndex($index->getProviderId(), $index->getDocumentId()); + $this->getIndex($index->getProviderId(), $index->getDocumentId(), $index->getCollection()); } catch (IndexDoesNotExistException $e) { return false; } @@ -86,7 +88,7 @@ class IndexesRequest extends IndexesRequestBuilder { $this->limitToProviderId($qb, $index->getProviderId()); $this->limitToDocumentId($qb, $index->getDocumentId()); - + $this->limitToCollection($qb, $index->getCollection()); $qb->execute(); return true; @@ -109,7 +111,6 @@ class IndexesRequest extends IndexesRequestBuilder { * @return Index[] */ public function getErrorIndexes(): array { - $qb = $this->getIndexesSelectSql(); $this->limitToErr($qb); @@ -129,36 +130,32 @@ class IndexesRequest extends IndexesRequestBuilder { * * @return bool */ - public function update(Index $index): bool { - - try { - $this->getIndex($index->getProviderId(), $index->getDocumentId()); - } catch (IndexDoesNotExistException $e) { - return false; - } - + public function update(Index $index, bool $statusOnly = false): bool { $qb = $this->getIndexesUpdateSql(); $qb->set('status', $qb->createNamedParameter($index->getStatus())); - $qb->set('source', $qb->createNamedParameter($index->getSource())); - $qb->set('options', $qb->createNamedParameter(json_encode($index->getOptions()))); - if ($index->getOwnerId() !== '') { - $qb->set('owner_id', $qb->createNamedParameter($index->getOwnerId())); - } + if (!$statusOnly) { + $qb->set('source', $qb->createNamedParameter($index->getSource())); - if ($index->getLastIndex() > 0) { - $qb->set('indexed', $qb->createNamedParameter($index->getLastIndex())); - } + $qb->set('options', $qb->createNamedParameter(json_encode($index->getOptions()))); + + if ($index->getOwnerId() !== '') { + $qb->set('owner_id', $qb->createNamedParameter($index->getOwnerId())); + } - $qb->set('message', $qb->createNamedParameter(json_encode($index->getErrors()))); - $qb->set('err', $qb->createNamedParameter($index->getErrorCount())); + if ($index->getLastIndex() > 0) { + $qb->set('indexed', $qb->createNamedParameter($index->getLastIndex())); + } + + $qb->set('message', $qb->createNamedParameter(json_encode($index->getErrors()))); + $qb->set('err', $qb->createNamedParameter($index->getErrorCount())); + } $this->limitToProviderId($qb, $index->getProviderId()); $this->limitToDocumentId($qb, $index->getDocumentId()); + $this->limitToCollection($qb, $index->getCollection()); - $qb->execute(); - - return true; + return ($qb->executeStatement() === 1); } @@ -167,27 +164,30 @@ class IndexesRequest extends IndexesRequestBuilder { * @param string $documentId * @param int $status */ - public function updateStatus(string $providerId, string $documentId, int $status) { + public function updateStatus(string $collection, string $providerId, string $documentId, int $status) { $qb = $this->getIndexesUpdateSql(); $qb->set('status', $qb->createNamedParameter($status)); $this->limitToProviderId($qb, $providerId); $this->limitToDocumentId($qb, $documentId); + $this->limitToCollection($qb, $collection); $qb->execute(); } /** + * @param string $collection * @param string $providerId * @param array $indexes * @param int $status */ - public function updateStatuses(string $providerId, array $indexes, int $status) { + public function updateStatuses(string $collection, string $providerId, array $indexes, int $status) { $qb = $this->getIndexesUpdateSql(); $qb->set('status', $qb->createNamedParameter($status)); $this->limitToProviderId($qb, $providerId); $this->limitToDocumentIds($qb, $indexes); + $this->limitToCollection($qb, $collection); $qb->execute(); } @@ -200,8 +200,20 @@ class IndexesRequest extends IndexesRequestBuilder { $qb = $this->getIndexesDeleteSql(); $this->limitToProviderId($qb, $index->getProviderId()); $this->limitToDocumentId($qb, $index->getDocumentId()); + $this->limitToCollection($qb, $index->getCollection()); - $qb->execute(); + $qb->executeStatement(); + } + + + /** + * @param string $collection \ + */ + public function deleteCollection(string $collection): void { + $qb = $this->getIndexesDeleteSql(); + $this->limitToCollection($qb, $collection); + + $qb->executeStatement(); } @@ -219,10 +231,11 @@ class IndexesRequest extends IndexesRequestBuilder { /** * */ - public function reset() { + public function reset(string $collection = ''): void { $qb = $this->getIndexesDeleteSql(); + $this->limitToCollection($qb, $collection); - $qb->execute(); + $qb->executeStatement(); } @@ -235,10 +248,11 @@ class IndexesRequest extends IndexesRequestBuilder { * @return Index * @throws IndexDoesNotExistException */ - public function getIndex(string $providerId, string $documentId): Index { + public function getIndex(string $providerId, string $documentId, string $collection = ''): Index { $qb = $this->getIndexesSelectSql(); $this->limitToProviderId($qb, $providerId); $this->limitToDocumentId($qb, $documentId); + $this->limitToCollection($qb, $collection); $cursor = $qb->execute(); $data = $cursor->fetch(); @@ -256,15 +270,14 @@ class IndexesRequest extends IndexesRequestBuilder { * return index. * * @param string $providerId - * @param array $documentIds + * @param string $documentId * * @return Index[] - * @throws IndexDoesNotExistException */ - public function getIndexes(string $providerId, array $documentIds): array { + public function getIndexes(string $providerId, string $documentId): array { $qb = $this->getIndexesSelectSql(); $this->limitToProviderId($qb, $providerId); - $this->limitToDocumentIds($qb, $documentIds); + $this->limitToDocumentId($qb, $documentId); $indexes = []; $cursor = $qb->execute(); @@ -273,10 +286,6 @@ class IndexesRequest extends IndexesRequestBuilder { } $cursor->closeCursor(); - if (sizeof($indexes) === 0) { - throw new IndexDoesNotExistException($this->l10n->t('Index not found')); - } - return $indexes; } @@ -286,13 +295,18 @@ class IndexesRequest extends IndexesRequestBuilder { * * @return Index[] */ - public function getQueuedIndexes(bool $all = false): array { + public function getQueuedIndexes(string $collection = '', bool $all = false, int $length = 0): array { $qb = $this->getIndexesSelectSql(); $this->limitToQueuedIndexes($qb); if ($all === false) { $this->limitToNoErr($qb); } + $this->limitToCollection($qb, $collection); + if ($length > 0) { + $qb->setMaxResults($length); + } + $indexes = []; $cursor = $qb->execute(); while ($data = $cursor->fetch()) { @@ -327,4 +341,69 @@ class IndexesRequest extends IndexesRequestBuilder { } + /** + * @return string[] + */ + public function getCollections(): array { + $qb = $this->dbConnection->getQueryBuilder(); + + $qb->select('li.collection') + ->from(self::TABLE_INDEXES, 'li'); + $qb->andWhere($qb->expr()->nonEmptyString('li.collection')); + $qb->groupBy('li.collection'); + + $collections = []; + $cursor = $qb->executeQuery(); + while ($data = $cursor->fetch()) { + $collections[] = $this->get('collection', $data); + } + $cursor->closeCursor(); + + return array_merge([''], $collections); + } + + + /** + * TODO: remove this in NC30+ + * + * @param string $providerId + * @param string $documentId + */ + public function migrateIndex24(string $providerId, string $documentId): array { + try { + $this->configService->requireMigration24(); + + return []; + } catch (DatabaseException $e) { + } + + // check data from old table. + /** @var IDBConnection $dbConnection */ + $dbConnection = \OC::$server->get(IDBConnection::class); + + $query = $dbConnection->getQueryBuilder(); + $query->select('*') + ->from('fulltextsearch_indexes') + ->where($query->expr()->eq('provider_id', $query->createNamedParameter($providerId))) + ->andWhere($query->expr()->eq('document_id', $query->createNamedParameter($documentId))); + + $result = $query->executeQuery(); + if ($result->rowCount() === 0) { + return []; + } + + $data = $result->fetch(); + $index = $this->parseIndexesSelectSql($data); + $result->closeCursor(); + + $query = $dbConnection->getQueryBuilder(); + $query->delete('fulltextsearch_indexes') + ->where($query->expr()->eq('provider_id', $query->createNamedParameter($providerId))) + ->andWhere($query->expr()->eq('document_id', $query->createNamedParameter($documentId))); + $query->executeStatement(); + + $this->create($index); + + return $this->getIndexes($providerId, $documentId); + } } diff --git a/lib/Db/IndexesRequestBuilder.php b/lib/Db/IndexesRequestBuilder.php index d81df09..7634ec6 100644 --- a/lib/Db/IndexesRequestBuilder.php +++ b/lib/Db/IndexesRequestBuilder.php @@ -83,8 +83,8 @@ class IndexesRequestBuilder extends CoreRequestBuilder { /** @noinspection PhpMethodParametersCountMismatchInspection */ $qb->select( - 'li.owner_id', 'li.provider_id', 'li.document_id', 'li.source', 'li.status', - 'li.options', 'li.err', 'li.message', 'li.indexed' + 'li.owner_id', 'li.provider_id', 'li.document_id', 'li.collection', 'li.source', + 'li.status', 'li.options', 'li.err', 'li.message', 'li.indexed' ) ->from(self::TABLE_INDEXES, 'li'); @@ -119,7 +119,8 @@ class IndexesRequestBuilder extends CoreRequestBuilder { $index->setStatus($this->getInt('status', $data)) ->setSource($this->get('source', $data, '')) ->setOwnerId($this->get('owner_id', $data, '')) - ->setLastIndex($this->getInt('indexed', $data, 0)); + ->setLastIndex($this->getInt('indexed', $data, 0)) + ->setCollection($this->get('collection', $data)); $index->setOptions($this->getArray('options', $data, [])); $index->setErrorCount($this->getInt('err', $data, 0)); $index->setErrors($this->getArray('message', $data, [])); diff --git a/lib/Exceptions/CollectionArgumentException.php b/lib/Exceptions/CollectionArgumentException.php new file mode 100644 index 0000000..eb05978 --- /dev/null +++ b/lib/Exceptions/CollectionArgumentException.php @@ -0,0 +1,40 @@ + + * @copyright 2018 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Exceptions; + + +use Exception; + + +class CollectionArgumentException extends Exception { + +} + diff --git a/lib/Migration/Version2000Date20201208130255.php b/lib/Migration/Version2000Date20201208130255.php index 01184dd..dc6d710 100644 --- a/lib/Migration/Version2000Date20201208130255.php +++ b/lib/Migration/Version2000Date20201208130255.php @@ -32,46 +32,6 @@ class Version2000Date20201208130255 extends SimpleMigrationStep { /** @var ISchemaWrapper $schema */ $schema = $schemaClosure(); - if (!$schema->hasTable('fulltextsearch_indexes')) { - $table = $schema->createTable('fulltextsearch_indexes'); - $table->addColumn('provider_id', 'string', [ - 'notnull' => true, - 'length' => 255, - ]); - $table->addColumn('document_id', 'string', [ - 'notnull' => true, - 'length' => 254, - ]); - $table->addColumn('source', 'string', [ - 'notnull' => false, - 'length' => 64, - ]); - $table->addColumn('owner_id', 'string', [ - 'notnull' => true, - 'length' => 64, - ]); - $table->addColumn('status', 'smallint', [ - 'notnull' => true, - 'length' => 1, - ]); - $table->addColumn('options', 'string', [ - 'notnull' => false, - 'length' => 511, - ]); - $table->addColumn('err', 'smallint', [ - 'notnull' => true, - 'length' => 1, - ]); - $table->addColumn('message', 'text', [ - 'notnull' => false, - ]); - $table->addColumn('indexed', 'bigint', [ - 'notnull' => false, - 'length' => 6, - ]); - $table->setPrimaryKey(['provider_id', 'document_id']); - } - if (!$schema->hasTable('fulltextsearch_ticks')) { $table = $schema->createTable('fulltextsearch_ticks'); $table->addColumn('id', 'bigint', [ diff --git a/lib/Migration/Version2400Date202201301329.php b/lib/Migration/Version2400Date202201301329.php new file mode 100644 index 0000000..0290be5 --- /dev/null +++ b/lib/Migration/Version2400Date202201301329.php @@ -0,0 +1,162 @@ +dbConnection = $dbConnection; + $this->configService = $configService; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + */ + public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + + if (!$schema->hasTable('fulltextsearch_index')) { + $table = $schema->createTable('fulltextsearch_index'); + $table->addColumn( + 'provider_id', 'string', + [ + 'notnull' => true, + 'length' => 254, + ] + ); + $table->addColumn( + 'document_id', 'string', + [ + 'notnull' => true, + 'length' => 254, + ] + ); + $table->addColumn( + 'collection', 'string', + [ + 'default' => '', + 'notnull' => false, + 'length' => 31 + ] + ); + $table->addColumn( + 'source', 'string', + [ + 'notnull' => false, + 'length' => 64, + ] + ); + $table->addColumn( + 'owner_id', 'string', + [ + 'notnull' => true, + 'length' => 64, + ] + ); + $table->addColumn( + 'status', 'smallint', + [ + 'notnull' => true, + 'length' => 1, + ] + ); + $table->addColumn( + 'options', 'string', + [ + 'notnull' => false, + 'length' => 511, + ] + ); + $table->addColumn( + 'err', 'smallint', + [ + 'notnull' => true, + 'length' => 1, + ] + ); + $table->addColumn( + 'message', 'text', + [ + 'notnull' => false, + ] + ); + $table->addColumn( + 'indexed', 'bigint', + [ + 'notnull' => false, + 'length' => 6, + 'unsigned' => true + ] + ); + $table->setPrimaryKey(['provider_id', 'document_id', 'collection']); + $table->addIndex(['collection']); + $table->addIndex(['collection', 'provider_id', 'document_id', 'status'], 'cpds'); + } + + return $schema; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * + * @throws Exception + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + if (!$schema->hasTable('fulltextsearch_indexes')) { + return; + } + + $query = $this->dbConnection->getQueryBuilder(); + $query->select($query->func()->count('*', 'index_count')) + ->from('fulltextsearch_indexes'); + $result = $query->executeQuery(); + $row = $result->fetch(); + $result->closeCursor(); + + if ((int)$row['index_count'] > 0) { + $this->configService->setAppValue(ConfigService::MIGRATION_24, '0'); + } + } +} diff --git a/lib/Model/Index.php b/lib/Model/Index.php index a324fff..31d74ec 100644 --- a/lib/Model/Index.php +++ b/lib/Model/Index.php @@ -34,6 +34,7 @@ namespace OCA\FullTextSearch\Model; use ArtificialOwl\MySmallPhpTools\Traits\TArrayTools; use JsonSerializable; use OCP\FullTextSearch\Model\IIndex; +use OCP\IURLGenerator; /** @@ -53,6 +54,9 @@ class Index implements IIndex, JsonSerializable { /** @var string */ private $documentId; + /** @var string */ + private $collection; + /** @var string */ private $source = ''; @@ -80,10 +84,12 @@ class Index implements IIndex, JsonSerializable { * * @param string $providerId * @param string $documentId + * @param string $collection */ - public function __construct(string $providerId, string $documentId) { + public function __construct(string $providerId, string $documentId, string $collection = '') { $this->providerId = $providerId; $this->documentId = $documentId; + $this->collection = $collection; } @@ -140,6 +146,25 @@ class Index implements IIndex, JsonSerializable { } + /** + * @param string $collection + * + * @return Index + */ + public function setCollection(string $collection): self { + $this->collection = $collection; + + return $this; + } + + /** + * @return string + */ + public function getCollection(): string { + return $this->collection; + } + + /** * @param int $status * @param bool $reset @@ -318,9 +343,9 @@ class Index implements IIndex, JsonSerializable { public function addError(string $message, string $exception = '', int $sev = IIndex::ERROR_SEV_3 ): IIndex { $this->errors[] = [ - 'message' => substr($message, 0, 1800), + 'message' => substr($message, 0, 1800), 'exception' => $exception, - 'severity' => $sev + 'severity' => $sev ]; $this->err++; @@ -352,20 +377,39 @@ class Index implements IIndex, JsonSerializable { } + /** + * @param IURLGenerator $urlGenerator + * + * @return array{url: string, status: int} + */ + public function asSitemap(IURLGenerator $urlGenerator): array { + return [ + 'url' => $urlGenerator->linkToOCSRouteAbsolute('fulltextsearch.Collection.indexDocument', + [ + 'collection' => $this->getCollection(), + 'providerId' => $this->getProviderId(), + 'documentId' => $this->getDocumentId() + ] + ), + 'status' => $this->getStatus() + ]; + } + /** * @return array */ public function jsonSerialize(): array { return [ - 'ownerId' => $this->getOwnerId(), + 'ownerId' => $this->getOwnerId(), 'providerId' => $this->getProviderId(), - 'source' => $this->getSource(), + 'collection' => $this->getCollection(), + 'source' => $this->getSource(), 'documentId' => $this->getDocumentId(), - 'lastIndex' => $this->getLastIndex(), - 'errors' => $this->getErrors(), + 'lastIndex' => $this->getLastIndex(), + 'errors' => $this->getErrors(), 'errorCount' => $this->getErrorCount(), - 'status' => (int)$this->getStatus(), - 'options' => $this->getOptions() + 'status' => (int)$this->getStatus(), + 'options' => $this->getOptions() ]; } diff --git a/lib/Model/Runner.php b/lib/Model/Runner.php index f09075e..0861861 100644 --- a/lib/Model/Runner.php +++ b/lib/Model/Runner.php @@ -210,6 +210,15 @@ class Runner implements IRunner { $this->infoUpdated(); } + /** + * @param string $info + * @param int $value + */ + public function setInfoInt(string $info, int $value): void { + $this->info[$info] = $value; + $this->infoUpdated(); + } + /** * @param array $data */ @@ -271,6 +280,14 @@ class Runner implements IRunner { return $this->get($info, $this->info, ''); } + /** + * @param string $info + * + * @return int + */ + public function getInfoInt(string $info): int { + return $this->getInt($info, $this->info); + } /** * @param array $method @@ -323,10 +340,10 @@ class Runner implements IRunner { public function newIndexError(IIndex $index, string $message, string $class = '', int $sev = 3 ) { $error = [ - 'index' => $index, - 'message' => $message, + 'index' => $index, + 'message' => $message, 'exception' => $class, - 'severity' => $sev + 'severity' => $sev ]; foreach ($this->methodOnIndexError as $method) { @@ -351,10 +368,10 @@ class Runner implements IRunner { */ public function newIndexResult(IIndex $index, string $message, string $status, int $type) { $result = [ - 'index' => $index, + 'index' => $index, 'message' => $message, - 'status' => $status, - 'type' => $type + 'status' => $status, + 'type' => $type ]; foreach ($this->methodOnIndexResult as $method) { diff --git a/lib/Service/CliService.php b/lib/Service/CliService.php index 6e08977..3b16977 100644 --- a/lib/Service/CliService.php +++ b/lib/Service/CliService.php @@ -170,7 +170,7 @@ class CliService { $initVar = $this->runner->getInfoAll(); $keys = array_keys($initVar); foreach ($keys as $key) { - $this->display->setMessage($initVar[$key], $key); + $this->display->setMessage((string)$initVar[$key], (string)$key); } $this->display->clear(); @@ -225,7 +225,7 @@ class CliService { $keys = array_keys($info); foreach ($keys as $k) { - $this->display->setMessage((string)$info[$k], $k); + $this->display->setMessage((string)$info[$k], (string)$k); } $this->refreshInfo(); diff --git a/lib/Service/CollectionService.php b/lib/Service/CollectionService.php new file mode 100644 index 0000000..4f18afb --- /dev/null +++ b/lib/Service/CollectionService.php @@ -0,0 +1,285 @@ + + * @copyright 2022 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Service; + + +use OCA\FullTextSearch\Db\IndexesRequest; +use OCA\FullTextSearch\Exceptions\CollectionArgumentException; +use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException; +use OCA\FullTextSearch\Exceptions\ProviderDoesNotExistException; +use OCA\FullTextSearch\Model\Index; +use OCA\FullTextSearch\Model\IndexOptions; +use OCA\FullTextSearch\Model\Runner; +use OCP\FullTextSearch\IFullTextSearchProvider; +use OCP\FullTextSearch\Model\IIndex; +use OCP\FullTextSearch\Model\IIndexDocument; +use OCP\IURLGenerator; + + +class CollectionService { + + + /** @var IURLGenerator */ + private $urlGenerator; + + /** @var IndexesRequest */ + private $indexesRequest; + + /** @var ProviderService */ + private $providerService; + + /** @var IndexService */ + private $indexService; + + /** @var ConfigService */ + private $configService; + + + /** @var Runner */ + private $runner; + + + /** + * @param IURLGenerator $urlGenerator + * @param IndexesRequest $indexesRequest + * @param ProviderService $providerService + * @param IndexService $indexService + * @param ConfigService $configService + */ + public function __construct( + IURLGenerator $urlGenerator, + IndexesRequest $indexesRequest, + ProviderService $providerService, + IndexService $indexService, + ConfigService $configService + ) { + $this->urlGenerator = $urlGenerator; + $this->indexesRequest = $indexesRequest; + $this->providerService = $providerService; + $this->indexService = $indexService; + $this->configService = $configService; + } + + + public function setRunner(Runner $runner): void { + $this->runner = $runner; + } + + + /** + * @param string $collection + */ + public function deleteCollection(string $collection): void { + $this->indexesRequest->deleteCollection($collection); + } + + + /** + * @param string $collection + * + * @return bool + */ + public function hasCollection(string $collection): bool { + foreach ($this->getCollections() as $item) { + if ($item === $collection) { + return true; + } + } + + return false; + } + + /** + * @return string[] + */ + public function getCollections(): array { + return array_filter($this->indexesRequest->getCollections()); + } + + + /** + * @param string $collection + * @param int $length + * + * @return array + */ + public function getQueue(string $collection, int $length = 0): array { + if ($length === 0) { + $length = $this->configService->getAppValueInt(ConfigService::COLLECTION_INDEXING_LIST); + } + + return array_map( + function (Index $index): array { + return $index->asSitemap($this->urlGenerator); + }, + $this->indexesRequest->getQueuedIndexes($collection, false, $length) + ); + } + + + /** + * @param string $collection + * @param string $providerId + * @param string $documentId + * + * @throws IndexDoesNotExistException + */ + public function setAsDone(string $collection, string $providerId, string $documentId): void { + $index = $this->indexesRequest->getIndex($providerId, $documentId, $collection); + $index->setStatus(IIndex::INDEX_DONE); + $this->indexService->updateIndex($index); + } + + + /** + * @param IFullTextSearchProvider $provider + * @param string $collection + * @param string $userId + * @param IndexOptions $options + */ + public function initCollectionIndexes( + IFullTextSearchProvider $provider, + string $collection, + string $userId, + IndexOptions $options + ) { + $chunks = $provider->generateChunks($userId); + if (empty($chunks)) { + $chunks = [$userId]; + } + + $this->updateRunnerInfo('chunkTotal', (string)sizeof($chunks)); + $chunkCount = 0; + foreach ($chunks as $chunk) { + $documents = $provider->generateIndexableDocuments($userId, (string)$chunk); + $this->updateRunnerInfoArray( + [ + 'chunkCurr' => ++$chunkCount, + 'documentChunk' => sizeof($documents) + ] + ); + + $documentCount = 0; + foreach ($documents as $document) { + $this->updateRunnerInfo('documentCurr', (string)++$documentCount); + $curr = $this->runner->getInfoInt('documentTotal'); + $this->runner->setInfoInt('documentTotal', ++$curr); + + try { + $this->indexesRequest->getIndex( + $document->getProviderId(), + $document->getId(), + $collection + ); + } catch (IndexDoesNotExistException $e) { + $index = new Index($document->getProviderId(), $document->getId(), $collection); + $index->setStatus(IIndex::INDEX_FULL); + $index->setOwnerId($document->getAccess()->getOwnerId()); + $index->setSource($document->getSource()); + $index->setLastIndex(); + $this->indexesRequest->create($index); + + $curr = $this->runner->getInfoInt('indexCount'); + $this->runner->setInfoInt('indexCount', ++$curr); + } + } + } + } + + + /** + * @param string $collection + * + * @throws CollectionArgumentException + */ + public function confirmCollectionString(string $collection): void { +// throw new CollectionArgumentException(); + } + + /** + * @param string $collection + * + * @throws CollectionArgumentException + */ + public function confirmCollection(string $collection): void { + $this->confirmCollectionString($collection); + + if (!$this->hasCollection($collection)) { + throw new CollectionArgumentException('collection does not exist'); + } + } + + + /** + * @param string $collection + * @param string $providerId + * @param string $documentId + * + * @return IIndexDocument + * @throws ProviderDoesNotExistException + * @throws IndexDoesNotExistException + */ + public function getDocument(string $collection, string $providerId, string $documentId): IIndexDocument { + $wrapped = $this->providerService->getProvider($providerId); + $provider = $wrapped->getProvider(); + + $index = $this->indexService->getIndex($providerId, $documentId, $collection); + $index->setStatus(IIndex::INDEX_FULL); + + return $provider->updateDocument($index); + } + + + /** + * @param string $info + * @param string $value + */ + private function updateRunnerInfo(string $info, string $value): void { + if (is_null($this->runner)) { + return; + } + + $this->runner->setInfo($info, $value); + } + + + /** + * @param array $data + */ + private function updateRunnerInfoArray(array $data): void { + if (is_null($this->runner)) { + return; + } + + $this->runner->setInfoArray($data); + } + +} diff --git a/lib/Service/ConfigService.php b/lib/Service/ConfigService.php index 457ec1a..1bd5a2e 100644 --- a/lib/Service/ConfigService.php +++ b/lib/Service/ConfigService.php @@ -32,6 +32,7 @@ namespace OCA\FullTextSearch\Service; use OCA\FullTextSearch\AppInfo\Application; +use OCA\FullTextSearch\Exceptions\DatabaseException; use OCA\FullTextSearch\Exceptions\ProviderOptionsDoesNotExistException; use OCP\IConfig; use OCP\PreConditionNotMetException; @@ -51,14 +52,21 @@ class ConfigService { const PROVIDER_INDEXED = 'provider_indexed'; const CRON_LAST_ERR_RESET = 'cron_err_reset'; const TICK_TTL = 'tick_ttl'; + const COLLECTION_INDEXING_LIST = 'collection_indexing_list'; + + + // Temp. can be removed after few major releases + const MIGRATION_24 = 'migration_24'; /** @var array */ public $defaults = [ - self::SEARCH_PLATFORM => '', - self::APP_NAVIGATION => '0', - self::PROVIDER_INDEXED => '', + self::SEARCH_PLATFORM => '', + self::APP_NAVIGATION => '0', + self::PROVIDER_INDEXED => '', self::CRON_LAST_ERR_RESET => '0', - self::TICK_TTL => '1800' + self::TICK_TTL => '1800', + self::COLLECTION_INDEXING_LIST => 50, + self::MIGRATION_24 => 1 ]; @@ -135,14 +143,24 @@ class ConfigService { * @return string */ public function getAppValue(string $key): string { - $defaultValue = null; + $defaultValue = ''; if (array_key_exists($key, $this->defaults)) { $defaultValue = $this->defaults[$key]; } - return $this->config->getAppValue(Application::APP_ID, $key, $defaultValue); + return (string)$this->config->getAppValue(Application::APP_ID, $key, $defaultValue); } + /** + * @param string $config + * + * @return int + */ + public function getAppValueInt(string $config): int { + return (int)$this->getAppValue($config); + } + + /** * Set a value by key * @@ -274,4 +292,16 @@ class ConfigService { return $ver[0]; } + + + /** + * @throws DatabaseException + */ + public function requireMigration24(): void { + if ($this->getAppValueInt(self::MIGRATION_24) === 1) { + return; + } + + throw new DatabaseException('please run ./occ fulltextsearch:migration:24'); + } } diff --git a/lib/Service/IndexService.php b/lib/Service/IndexService.php index 2192536..b30283b 100644 --- a/lib/Service/IndexService.php +++ b/lib/Service/IndexService.php @@ -33,9 +33,9 @@ namespace OCA\FullTextSearch\Service; use Exception; use OCA\FullTextSearch\Db\IndexesRequest; -use OCA\FullTextSearch\Exceptions\DatabaseException; use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException; use OCA\FullTextSearch\Exceptions\NotIndexableDocumentException; +use OCA\FullTextSearch\Exceptions\ProviderDoesNotExistException; use OCA\FullTextSearch\Model\Index; use OCA\FullTextSearch\Model\IndexOptions; use OCA\FullTextSearch\Model\Runner; @@ -59,18 +59,12 @@ class IndexService implements IIndexService { /** @var IndexesRequest */ private $indexesRequest; - /** @var ConfigService */ - private $configService; - /** @var ProviderService */ private $providerService; /** @var PlatformService */ private $platformService; - /** @var MiscService */ - private $miscService; - /** @var Runner */ private $runner = null; @@ -86,20 +80,17 @@ class IndexService implements IIndexService { * IndexService constructor. * * @param IndexesRequest $indexesRequest - * @param ConfigService $configService * @param ProviderService $providerService * @param PlatformService $platformService - * @param MiscService $miscService */ public function __construct( - IndexesRequest $indexesRequest, ConfigService $configService, - ProviderService $providerService, PlatformService $platformService, MiscService $miscService + IndexesRequest $indexesRequest, + ProviderService $providerService, + PlatformService $platformService ) { $this->indexesRequest = $indexesRequest; - $this->configService = $configService; $this->providerService = $providerService; $this->platformService = $platformService; - $this->miscService = $miscService; } @@ -167,15 +158,15 @@ class IndexService implements IIndexService { $this->updateRunnerAction('generateIndex' . $provider->getName()); $this->updateRunnerInfoArray( [ - 'userId' => $userId, - 'providerId' => $provider->getId(), - 'providerName' => $provider->getName(), - 'chunkCurrent' => 0, - 'chunkTotal' => 0, + 'userId' => $userId, + 'providerId' => $provider->getId(), + 'providerName' => $provider->getName(), + 'chunkCurrent' => 0, + 'chunkTotal' => 0, 'documentCurrent' => 0, - 'documentTotal' => 0, - 'info' => '', - 'title' => '' + 'documentTotal' => 0, + 'info' => '', + 'title' => '' ] ); @@ -193,14 +184,14 @@ class IndexService implements IIndexService { $this->currentTotalDocuments = sizeof($documents); $this->updateRunnerInfoArray( [ - 'documentTotal' => $this->currentTotalDocuments, + 'documentTotal' => $this->currentTotalDocuments, 'documentCurrent' => 0 ] ); //$maxSize = sizeof($documents); - $toIndex = $this->updateDocumentsWithCurrIndex($provider, $documents, $options); + $toIndex = $this->updateDocumentsWithCurrIndex($provider, '', $documents, $options); $this->indexDocuments($platform, $provider, $toIndex, $options); } } @@ -214,14 +205,16 @@ class IndexService implements IIndexService { * @return IIndexDocument[] * @throws Exception */ - private function updateDocumentsWithCurrIndex( - IFullTextSearchProvider $provider, array $documents, IIndexOptions $options + public function updateDocumentsWithCurrIndex( + IFullTextSearchProvider $provider, + string $collection, + array $documents, + IIndexOptions $options ): array { $result = []; $count = 0; foreach ($documents as $document) { - if ($count % 1000 === 0) { $this->updateRunnerAction('compareWithCurrentIndex', true); $this->updateRunnerInfo('documentCurrent', (string)$count); @@ -229,10 +222,13 @@ class IndexService implements IIndexService { $count++; try { - $index = - $this->indexesRequest->getIndex($document->getProviderId(), $document->getId()); + $index = $this->indexesRequest->getIndex( + $document->getProviderId(), + $document->getId(), + $collection + ); } catch (IndexDoesNotExistException $e) { - $index = new Index($document->getProviderId(), $document->getId()); + $index = new Index($document->getProviderId(), $document->getId(), $collection); $index->setStatus(Index::INDEX_FULL); $index->setLastIndex(); } @@ -267,7 +263,9 @@ class IndexService implements IIndexService { * * @return bool */ - private function isDocumentUpToDate(IFullTextSearchProvider $provider, IIndexDocument $document + private function isDocumentUpToDate( + IFullTextSearchProvider $provider, + IIndexDocument $document ): bool { $index = $document->getIndex(); if (!$index->isStatus(Index::INDEX_OK)) { @@ -305,11 +303,11 @@ class IndexService implements IIndexService { $this->updateRunnerAction('fillDocument', true); $this->updateRunnerInfoArray( [ - 'documentId' => $document->getId(), - 'info' => '', - 'title' => '', - 'content' => '', - 'status' => '', + 'documentId' => $document->getId(), + 'info' => '', + 'title' => '', + 'content' => '', + 'status' => '', 'statusColored' => '' ] ); @@ -317,7 +315,7 @@ class IndexService implements IIndexService { $provider->fillIndexDocument($document); $this->updateRunnerInfoArray( [ - 'title' => $document->getTitle(), + 'title' => $document->getTitle(), 'content' => $document->getContentSize() ] ); @@ -366,8 +364,8 @@ class IndexService implements IIndexService { $this->updateRunnerInfoArray( [ 'documentId' => $document->getId(), - 'title' => $document->getTitle(), - 'content' => $document->getContentSize() + 'title' => $document->getTitle(), + 'content' => $document->getContentSize() ] ); @@ -394,7 +392,7 @@ class IndexService implements IIndexService { $this->updateRunnerInfoArray( [ 'providerName' => $provider->getName(), - 'userId' => $index->getOwnerId(), + 'userId' => $index->getOwnerId(), ] ); @@ -424,8 +422,8 @@ class IndexService implements IIndexService { $this->updateRunnerInfoArray( [ 'documentId' => $document->getId(), - 'title' => $document->getTitle(), - 'content' => $document->getContentSize() + 'title' => $document->getTitle(), + 'content' => $document->getContentSize() ] ); @@ -438,28 +436,19 @@ class IndexService implements IIndexService { /** * @param Index[] $indexes - * - * @throws DatabaseException */ public function updateIndexes(array $indexes) { - try { - foreach ($indexes as $index) { - $this->updateIndex($index); - } - $this->resetErrorFromQueue(); - } catch (Exception $e) { - throw new DatabaseException($e->getMessage()); + foreach ($indexes as $index) { + $this->updateIndex($index); } + $this->resetErrorFromQueue(); } /** * @param IIndex $index - * - * @throws Exception */ public function updateIndex(IIndex $index) { - /** @var Index $index */ $this->updateIndexError($index); if ($index->isStatus(IIndex::INDEX_REMOVE)) { @@ -502,22 +491,20 @@ class IndexService implements IIndexService { * @throws Exception */ public function updateIndexStatus( - string $providerId, string $documentId, int $status, bool $reset = false + string $providerId, + string $documentId, + int $status, + bool $reset = false ) { - if ($reset === true) { - $this->indexesRequest->updateStatus($providerId, $documentId, $status); - - return; + $indexes = $this->indexesRequest->getIndexes($providerId, $documentId); + if (empty($indexes)) { + $indexes = $this->indexesRequest->migrateIndex24($providerId, $documentId); } - try { - $curr = $this->getIndex($providerId, $documentId); - } catch (IndexDoesNotExistException $e) { - return; + foreach ($indexes as $index) { + $index->setStatus($status); + $this->updateIndex($index); } - - $curr->setStatus($status); - $this->updateIndex($curr); } @@ -526,27 +513,18 @@ class IndexService implements IIndexService { * @param array $documentIds * @param int $status * @param bool $reset - * - * @throws DatabaseException */ public function updateIndexesStatus( - string $providerId, array $documentIds, int $status, bool $reset = false + string $providerId, + array $documentIds, + int $status, + bool $reset = false ) { - if ($reset === true) { - $this->indexesRequest->updateStatuses($providerId, $documentIds, $status); - - return; - } - - try { - $all = $this->getIndexes($providerId, $documentIds); - } catch (IndexDoesNotExistException $e) { - return; - } - - foreach ($all as $curr) { - $curr->setStatus($status); - $this->updateIndexes([$curr]); + foreach ($documentIds as $documentId) { + try { + $this->updateIndexStatus($providerId, $documentId, $status, $reset); + } catch (Exception $e) { + } } } @@ -588,39 +566,41 @@ class IndexService implements IIndexService { /** * @param string $providerId - * @param array $documentId + * @param string $documentId * * @return Index[] - * @throws IndexDoesNotExistException */ - public function getIndexes($providerId, $documentId) { + public function getIndexes(string $providerId, string $documentId): array { return $this->indexesRequest->getIndexes($providerId, $documentId); } /** + * @param string $collection * @param bool $all + * @param int $length * * @return Index[] */ - public function getQueuedIndexes(bool $all = false): array { - return $this->indexesRequest->getQueuedIndexes($all); + public function getQueuedIndexes(string $collection = '', bool $all = false, int $length = -1): array { + return $this->indexesRequest->getQueuedIndexes($collection, $all, $length); } /** * @param string $providerId + * @param string $collection * - * @throws Exception + * @throws ProviderDoesNotExistException */ - public function resetIndex(string $providerId = '') { + public function resetIndex(string $providerId = '', string $collection = '') { $wrapper = $this->platformService->getPlatform(); $platform = $wrapper->getPlatform(); if ($providerId === '') { $platform->resetIndex('all'); $this->providerService->setProvidersAsNotIndexed(); - $this->indexesRequest->reset(); + $this->indexesRequest->reset($collection); return; } else { @@ -628,13 +608,13 @@ class IndexService implements IIndexService { $providers = [$providerWrapper->getProvider()]; } - foreach ($providers AS $provider) { + foreach ($providers as $provider) { // TODO: need to specify the map to remove // TODO: need to remove entries with type=providerId // $provider->onResettingIndex($platform); $platform->resetIndex($provider->getId()); - $this->providerService->setProviderAsIndexed($provider, false); + $this->providerService->setProviderAsIndexed($provider->getId(), false); $this->indexesRequest->deleteFromProviderId($provider->getId()); } } @@ -647,8 +627,8 @@ class IndexService implements IIndexService { * @return IIndex * @throws IndexDoesNotExistException */ - public function getIndex(string $providerId, string $documentId): IIndex { - return $this->indexesRequest->getIndex($providerId, $documentId); + public function getIndex(string $providerId, string $documentId, string $collection = ''): IIndex { + return $this->indexesRequest->getIndex($providerId, $documentId, $collection); } @@ -659,23 +639,33 @@ class IndexService implements IIndexService { * @param int $status * * @return IIndex + * @throws IndexDoesNotExistException */ - public function createIndex(string $providerId, string $documentId, string $userId, int $status + public function createIndex( + string $providerId, + string $documentId, + string $userId, + int $status ): IIndex { + $index = null; + foreach ($this->indexesRequest->getCollections() as $collection) { + try { + $index = $this->indexesRequest->getIndex($providerId, $documentId, $collection); + $index->setStatus($status, true); + $this->indexesRequest->update($index, true); + } catch (IndexDoesNotExistException $e) { + $index = new Index($providerId, $documentId, $collection); + $index->setOwnerId($userId); + $index->setStatus($status); + $this->indexesRequest->create($index); + } + } - try { - $known = $this->indexesRequest->getIndex($providerId, $documentId); - $known->setStatus($status); - $this->indexesRequest->updateStatus($providerId, $documentId, $status); - - return $known; - } catch (IndexDoesNotExistException $e) { + if (is_null($index)) { + throw new IndexDoesNotExistException(); } - $index = new Index($providerId, $documentId); - $index->setOwnerId($userId); - $index->setStatus($status); - $this->indexesRequest->create($index); + $index->setCollection(''); return $index; } diff --git a/lib/Service/MigrationService.php b/lib/Service/MigrationService.php new file mode 100644 index 0000000..4b3c793 --- /dev/null +++ b/lib/Service/MigrationService.php @@ -0,0 +1,182 @@ + + * @copyright 2022 + * @license GNU AGPL version 3 or any later version + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + + +namespace OCA\FullTextSearch\Service; + + +use OCP\DB\Exception; +use OCP\DB\IResult; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; +use Symfony\Component\Console\Helper\ProgressBar; + + +class MigrationService { + + + /** @var IDBConnection */ + private $dbConnection; + + /** @var ConfigService */ + private $configService; + + + /** @var ProgressBar */ + private $progressBar; + + + /** + * @param IDBConnection $dbConnection + * @param ConfigService $configService + */ + public function __construct( + IDBConnection $dbConnection, + ConfigService $configService + ) { + $this->dbConnection = $dbConnection; + $this->configService = $configService; + } + + + public function setProgressBar(ProgressBar $progressBar): void { + $this->progressBar = $progressBar; + } + + + /** + * @param int $size + * + * @return bool + * @throws Exception + */ + public function migrate24Chunk(int $size = 10000): bool { + if ($this->configService->getAppValueInt(ConfigService::MIGRATION_24) === 1) { + return false; + } + + if ($size === 0) { + return false; + } + + $oldData = $this->getChunkOldData($size); + if ($oldData->rowCount() === 0) { + $this->configService->setAppValue(ConfigService::MIGRATION_24, '1'); + if ($this->dbConnection->tableExists('fulltextsearch_indexes')) { + $this->dbConnection->dropTable('fulltextsearch_indexes'); + } + + return false; + } + + $this->createNewEntries($oldData); + + return true; + } + + + /** + * @param int $size + * + * @return IResult + * @throws Exception + */ + private function getChunkOldData(int $size): IResult { + $query = $this->dbConnection->getQueryBuilder(); + $query->select('*') + ->from('fulltextsearch_indexes') + ->orderBy('provider_id', 'asc') + ->addOrderBy('document_id', 'asc') + ->setFirstResult(0) + ->setMaxResults($size); + + return $query->executeQuery(); + } + + + /** + * @param IResult $oldData + * + * @throws Exception + */ + private function createNewEntries(IResult $oldData): void { + $create = $this->dbConnection->getQueryBuilder(); + $create->insert('fulltextsearch_index') + ->values( + [ + 'owner_id' => $create->createParameter('owner_id'), + 'provider_id' => $create->createParameter('provider_id'), + 'collection' => $create->createParameter('collection'), + 'document_id' => $create->createParameter('document_id'), + 'source' => $create->createParameter('source'), + 'err' => $create->createParameter('err'), + 'message' => $create->createParameter('message'), + 'status' => $create->createParameter('status'), + 'options' => $create->createParameter('options'), + 'indexed' => $create->createParameter('indexed') + ] + ); + + while ($row = $oldData->fetch()) { + $create->setParameter('owner_id', $row['owner_id']) + ->setParameter('provider_id', $row['provider_id']) + ->setParameter('collection', '') + ->setParameter('document_id', $row['document_id']) + ->setParameter('source', $row['source']) + ->setParameter('err', $row['err'], IQueryBuilder::PARAM_INT) + ->setParameter('message', $row['message']) + ->setParameter('status', $row['status'], IQueryBuilder::PARAM_INT) + ->setParameter('options', $row['options']) + ->setParameter('indexed', $row['indexed'], IQueryBuilder::PARAM_INT); + + if (!is_null($this->progressBar)) { + $this->progressBar->advance(); + } + + try { + $create->executeStatement(); + } catch (Exception $e) { + if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) { + throw $e; + } + } + + $delete = $this->dbConnection->getQueryBuilder(); + $delete->delete('fulltextsearch_indexes'); + $delete->where( + $delete->expr()->eq('provider_id', $delete->createNamedParameter($row['provider_id'])) + ); + $delete->andWhere( + $delete->expr()->eq('document_id', $delete->createNamedParameter($row['document_id'])) + ); + + $delete->executeStatement(); + } + } +} diff --git a/lib/Service/ProviderService.php b/lib/Service/ProviderService.php index 17cda2d..093cfb4 100644 --- a/lib/Service/ProviderService.php +++ b/lib/Service/ProviderService.php @@ -244,12 +244,13 @@ class ProviderService implements IProviderService { /** - * @param IFullTextSearchProvider $provider + * @param string $providerId * @param bool $boolean */ - public function setProviderAsIndexed(IFullTextSearchProvider $provider, bool $boolean) { + public function setProviderAsIndexed(string $providerId, bool $boolean) { $this->configService->setProviderOptions( - $provider->getId(), ConfigService::PROVIDER_INDEXED, (($boolean) ? '1' : '0') + $providerId, + ConfigService::PROVIDER_INDEXED, (($boolean) ? '1' : '0') ); } @@ -291,7 +292,7 @@ class ProviderService implements IProviderService { if (array_key_exists('@attributes', $providers)) { $providers = [$providers]; } - foreach ($providers AS $provider) { + foreach ($providers as $provider) { if (is_array($provider)) { $attributes = $provider['@attributes']; if (array_key_exists('min-version', $attributes) @@ -326,7 +327,7 @@ class ProviderService implements IProviderService { * @throws Exception */ private function providerIdMustBeUnique(IFullTextSearchProvider $provider) { - foreach ($this->providers AS $providerWrapper) { + foreach ($this->providers as $providerWrapper) { $knownProvider = $providerWrapper->getProvider(); if ($knownProvider->getId() === $provider->getId()) { throw new ProviderIsNotUniqueException( @@ -346,7 +347,7 @@ class ProviderService implements IProviderService { $arr = []; foreach ($providers as $provider) { $arr[] = [ - 'id' => $provider->getId(), + 'id' => $provider->getId(), 'name' => $provider->getName() ]; } -- cgit v1.2.3