collections and external APIenh/noid/collections

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>
Co-authored-by: Carl Schwan <carl@carlschwan.eu>

25 files changed, 1850 insertions, 290 deletions

diff --git a/appinfo/info.xml b/appinfo/info.xml
index 16ae8e5..4dbd150 100644
--- a/appinfo/info.xml
+++ b/appinfo/info.xml
@@ -28,10 +28,14 @@ Core App of the full-text search framework for your Nextcloud.
 
 	<background-jobs>
 		<job>OCA\FullTextSearch\Cron\Index</job>
+		<job>OCA\FullTextSearch\Cron\Maintenance</job>
 	</background-jobs>
 
 	<commands>
 		<command>OCA\FullTextSearch\Command\Check</command>
+		<command>OCA\FullTextSearch\Command\CollectionInit</command>
+		<command>OCA\FullTextSearch\Command\CollectionDelete</command>
+		<command>OCA\FullTextSearch\Command\CollectionList</command>
 		<command>OCA\FullTextSearch\Command\Configure</command>
 		<command>OCA\FullTextSearch\Command\DocumentIndex</command>
 		<command>OCA\FullTextSearch\Command\DocumentPlatform</command>
@@ -39,6 +43,7 @@ Core App of the full-text search framework for your Nextcloud.
 		<command>OCA\FullTextSearch\Command\DocumentStatus</command>
 		<command>OCA\FullTextSearch\Command\Index</command>
 		<command>OCA\FullTextSearch\Command\Live</command>
+		<command>OCA\FullTextSearch\Command\Migration24</command>
 		<command>OCA\FullTextSearch\Command\Reset</command>
 		<command>OCA\FullTextSearch\Command\Search</command>
 		<command>OCA\FullTextSearch\Command\Stop</command>
diff --git a/appinfo/routes.php b/appinfo/routes.php
index 967ac82..5b56710 100644
--- a/appinfo/routes.php
+++ b/appinfo/routes.php
@@ -29,6 +29,21 @@ declare(strict_types=1);
 
 
 return [
+	'ocs' => [
+		/** @see OCA\FullTextSearch\Controller\CollectionController */
+		['name' => 'Collection#getQueue', 'url' => '/collection/{collection}/index', 'verb' => 'GET'],
+		[
+			'name' => 'Collection#indexDocument',
+			'url' => '/collection/{collection}/document/{providerId}/{documentId}',
+			'verb' => 'GET'
+		],
+		[
+			'name' => 'Collection#updateStatusDone',
+			'url' => '/collection/{collection}/document/{providerId}/{documentId}/done',
+			'verb' => 'POST'
+		]
+	],
+
 	'routes' => [
 		['name' => 'Navigation#navigate', 'url' => '/', 'verb' => 'GET'],
 		['name' => 'Settings#getSettingsAdmin', 'url' => '/admin/settings', 'verb' => 'GET'],
diff --git a/lib/Command/CollectionDelete.php b/lib/Command/CollectionDelete.php
new file mode 100644
index 0000000..98b0a23
--- /dev/null
+++ b/lib/Command/CollectionDelete.php
@@ -0,0 +1,89 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Command;
+
+
+use OC\Core\Command\Base;
+use OCA\FullTextSearch\Exceptions\CollectionArgumentException;
+use OCA\FullTextSearch\Service\CollectionService;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+class CollectionDelete extends Base {
+
+
+	/** @var CollectionService */
+	private $collectionService;
+
+
+	/**
+	 * @param CollectionService $collectionService
+	 */
+	public function __construct(CollectionService $collectionService) {
+		parent::__construct();
+
+		$this->collectionService = $collectionService;
+	}
+
+
+	/**
+	 *
+	 */
+	protected function configure() {
+		parent::configure();
+		$this->setName('fulltextsearch:collection:delete')
+			 ->setDescription('Delete collection')
+			 ->addArgument('name', InputArgument::REQUIRED, 'name of the collection to delete');
+	}
+
+
+	/**
+	 * @param InputInterface $input
+	 * @param OutputInterface $output
+	 *
+	 * @return int
+	 */
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$collection = $input->getArgument('name');
+		if (!$this->collectionService->hasCollection($collection)) {
+			throw new CollectionArgumentException('unknown collection');
+		}
+
+		$this->collectionService->deleteCollection($collection);
+
+		return 0;
+	}
+}
+
+
+
diff --git a/lib/Command/CollectionInit.php b/lib/Command/CollectionInit.php
new file mode 100644
index 0000000..3f5c263
--- /dev/null
+++ b/lib/Command/CollectionInit.php
@@ -0,0 +1,198 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Command;
+
+
+use Exception;
+use OC\Core\Command\Base;
+use OCA\FullTextSearch\Model\IndexOptions;
+use OCA\FullTextSearch\Model\Runner;
+use OCA\FullTextSearch\Service\CliService;
+use OCA\FullTextSearch\Service\CollectionService;
+use OCA\FullTextSearch\Service\ProviderService;
+use OCA\FullTextSearch\Service\RunningService;
+use OCP\FullTextSearch\IFullTextSearchProvider;
+use OCP\IUserManager;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+class CollectionInit extends Base {
+
+
+	/** @var ProviderService */
+	private $providerService;
+
+	/** @var CollectionService */
+	private $collectionService;
+
+	/** @var IUserManager */
+	private $userManager;
+
+	/** @var RunningService */
+	private $runningService;
+
+	/** @var CliService */
+	private $cliService;
+
+
+	/**
+	 * @param IUserManager $userManager
+	 * @param CollectionService $collectionService
+	 * @param ProviderService $providerService
+	 * @param RunningService $runningService
+	 * @param CliService $cliService
+	 */
+	public function __construct(
+		IUserManager $userManager,
+		CollectionService $collectionService,
+		ProviderService $providerService,
+		RunningService $runningService,
+		CliService $cliService
+	) {
+		parent::__construct();
+
+		$this->userManager = $userManager;
+		$this->collectionService = $collectionService;
+		$this->providerService = $providerService;
+		$this->runningService = $runningService;
+		$this->cliService = $cliService;
+	}
+
+
+	/**
+	 *
+	 */
+	protected function configure() {
+		parent::configure();
+		$this->setName('fulltextsearch:collection:init')
+			 ->setDescription('Initiate a collection')
+			 ->addArgument('name', InputArgument::REQUIRED, 'name of the collection');
+	}
+
+
+	/**
+	 * @param InputInterface $input
+	 * @param OutputInterface $output
+	 *
+	 * @return int
+	 * @throws Exception
+	 */
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$collection = $input->getArgument('name');
+		$this->collectionService->confirmCollectionString($collection);
+
+		$runner = new Runner($this->runningService, 'commandIndex', ['nextStep' => 'n']);
+//		$runner->sourceIsCommandLine($this, $output);
+		$this->collectionService->setRunner($runner);
+		$this->cliService->setRunner($runner);
+
+		$this->cliService->createPanel(
+			'collection', [
+							'┌─ Collection ' . $collection . ' ────',
+							'│ ProviderId, UserId: <info>%providerId%</info> / <info>%userId%</info>',
+							'│ Chunk: <info>%chunkCurr:3s%</info>/<info>%chunkTotal%</info>',
+							'│ Document: <info>%documentCurr:6s%</info>/<info>%documentChunk%</info>',
+							'│',
+							'│ Total Document: <info>%documentTotal%</info>',
+							'│ Index initiated: <info>%indexCount%</info>',
+							'└──'
+						]
+		);
+
+		$runner->setInfoArray([
+								  'providerId' => '',
+								  'userId' => '',
+								  'chunkCurr' => '',
+								  'chunkTotal' => '',
+								  'documentCurr' => '',
+								  'documentChunk' => '',
+								  'documentTotal' => '',
+								  'indexCount' => 0
+							  ]);
+
+		$this->cliService->initDisplay();
+		$this->cliService->displayPanel('run', 'collection');
+		$this->cliService->runDisplay($output);
+
+		$providers = $this->providerService->getProviders();
+		foreach ($providers as $providerWrapper) {
+			$this->indexProvider($runner, $collection, $providerWrapper->getProvider());
+		}
+
+
+		return 0;
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param IFullTextSearchProvider $provider
+	 *
+	 * @throws Exception
+	 */
+	private function indexProvider(
+		Runner $runner,
+		string $collection,
+		IFullTextSearchProvider $provider,
+		string $userId = ''
+	) {
+		$runner->setInfo('providerId', $provider->getId());
+		$options = new IndexOptions();
+		$provider->setIndexOptions($options);
+
+		if ($userId === '') {
+			$users = $this->userManager->search('');
+		} else {
+			$users = [$userId];
+		}
+
+		foreach ($users as $user) {
+			if ($user === null) {
+				continue;
+			}
+
+			$runner->setInfo('userId', $user->getUID());
+
+			try {
+				$this->collectionService->initCollectionIndexes(
+					$provider,
+					$collection,
+					$user->getUID(),
+					$options
+				);
+			} catch (Exception $e) {
+				continue;
+			}
+		}
+	}
+}
diff --git a/lib/Command/CollectionList.php b/lib/Command/CollectionList.php
new file mode 100644
index 0000000..c267107
--- /dev/null
+++ b/lib/Command/CollectionList.php
@@ -0,0 +1,83 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Command;
+
+
+use OC\Core\Command\Base;
+use OCA\FullTextSearch\Service\CollectionService;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+class CollectionList extends Base {
+
+
+	/** @var CollectionService */
+	private $collectionService;
+
+
+	/**
+	 * @param CollectionService $collectionService
+	 */
+	public function __construct(CollectionService $collectionService) {
+		parent::__construct();
+
+		$this->collectionService = $collectionService;
+	}
+
+
+	/**
+	 *
+	 */
+	protected function configure() {
+		parent::configure();
+		$this->setName('fulltextsearch:collection:list')
+			 ->setDescription('List collections');
+	}
+
+
+	/**
+	 * @param InputInterface $input
+	 * @param OutputInterface $output
+	 *
+	 * @return int
+	 */
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		$collections = $this->collectionService->getCollections();
+		$output->writeln('found ' . sizeof($collections) . ' collection(s)');
+
+		foreach ($this->collectionService->getCollections() as $collection) {
+			$output->writeln('- ' . $collection);
+		}
+
+		return 0;
+	}
+}
diff --git a/lib/Command/Index.php b/lib/Command/Index.php
index 73211f2..1508769 100644
--- a/lib/Command/Index.php
+++ b/lib/Command/Index.php
@@ -40,6 +40,7 @@ use OCA\FullTextSearch\Model\Index as ModelIndex;
 use OCA\FullTextSearch\Model\IndexOptions;
 use OCA\FullTextSearch\Model\Runner;
 use OCA\FullTextSearch\Service\CliService;
+use OCA\FullTextSearch\Service\ConfigService;
 use OCA\FullTextSearch\Service\IndexService;
 use OCA\FullTextSearch\Service\MiscService;
 use OCA\FullTextSearch\Service\PlatformService;
@@ -48,7 +49,6 @@ use OCA\FullTextSearch\Service\RunningService;
 use OCP\FullTextSearch\IFullTextSearchProvider;
 use OCP\IUserManager;
 use OutOfBoundsException;
-use Symfony\Component\Console\Formatter\OutputFormatterStyle;
 use Symfony\Component\Console\Input\InputArgument;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -138,6 +138,8 @@ class Index extends ACommandBase {
 	/** @var MiscService */
 	private $miscService;
 
+	/** @var ConfigService */
+	private $configService;
 
 	/** @var Runner */
 	private $runner;
@@ -168,11 +170,13 @@ class Index extends ACommandBase {
 	 * @param PlatformService $platformService
 	 * @param ProviderService $providerService
 	 * @param MiscService $miscService
+	 * @param ConfigService $configService
 	 */
 	public function __construct(
 		IUserManager $userManager, RunningService $runningService, CliService $cliService,
 		IndexService $indexService, PlatformService $platformService,
-		ProviderService $providerService, MiscService $miscService
+		ProviderService $providerService, MiscService $miscService,
+		ConfigService $configService
 	) {
 		parent::__construct();
 		$this->userManager = $userManager;
@@ -184,6 +188,7 @@ class Index extends ACommandBase {
 		$this->platformService = $platformService;
 		$this->providerService = $providerService;
 		$this->miscService = $miscService;
+		$this->configService = $configService;
 	}
 
 
@@ -210,6 +215,7 @@ class Index extends ACommandBase {
 	 * @throws Exception
 	 */
 	protected function execute(InputInterface $input, OutputInterface $output) {
+		$this->configService->requireMigration24();
 
 		$options = $this->generateIndexOptions($input);
 
@@ -217,7 +223,7 @@ class Index extends ACommandBase {
 			/** do not get stuck while waiting interactive input */
 			try {
 				readline_callback_handler_install(
-					'', function() {
+					'', function () {
 				}
 				);
 			} catch (Throwable $t) {
@@ -414,7 +420,7 @@ class Index extends ACommandBase {
 			}
 		}
 
-		$this->providerService->setProviderAsIndexed($provider, true);
+		$this->providerService->setProviderAsIndexed($provider->getId(), true);
 	}
 
 
@@ -572,37 +578,37 @@ class Index extends ACommandBase {
 		// full list of info that can be edited
 		$this->runner->setInfoArray(
 			[
-				'userId'       => '',
+				'userId' => '',
 				'providerName' => '',
-				'_memory'      => '',
-				'documentId'   => '',
-				'action'       => '',
-				'info'         => '',
-				'title'        => '',
-				'_paused'      => '',
-
-				'resultIndex'         => '',
-				'resultCurrent'       => '',
-				'resultTotal'         => '',
-				'resultMessageA'      => '',
-				'resultMessageB'      => '',
-				'resultMessageC'      => '',
-				'resultStatus'        => '',
+				'_memory' => '',
+				'documentId' => '',
+				'action' => '',
+				'info' => '',
+				'title' => '',
+				'_paused' => '',
+
+				'resultIndex' => '',
+				'resultCurrent' => '',
+				'resultTotal' => '',
+				'resultMessageA' => '',
+				'resultMessageB' => '',
+				'resultMessageC' => '',
+				'resultStatus' => '',
 				'resultStatusColored' => '',
-				'content'             => '',
-				'statusColored'       => '',
-				'chunkCurrent'        => '',
-				'chunkTotal'          => '',
-				'documentCurrent'     => '',
-				'documentTotal'       => '',
-				'progressStatus'      => '',
-				'errorCurrent'        => '0',
-				'errorTotal'          => '0',
-				'errorMessageA'       => '',
-				'errorMessageB'       => '',
-				'errorMessageC'       => '',
-				'errorException'      => '',
-				'errorIndex'          => ''
+				'content' => '',
+				'statusColored' => '',
+				'chunkCurrent' => '',
+				'chunkTotal' => '',
+				'documentCurrent' => '',
+				'documentTotal' => '',
+				'progressStatus' => '',
+				'errorCurrent' => '0',
+				'errorTotal' => '0',
+				'errorMessageA' => '',
+				'errorMessageB' => '',
+				'errorMessageC' => '',
+				'errorException' => '',
+				'errorIndex' => ''
 			]
 		);
 	}
@@ -618,7 +624,7 @@ class Index extends ACommandBase {
 			$this->runner->setInfoArray(
 				[
 					'errorCurrent' => 0,
-					'errorTotal'   => 0,
+					'errorTotal' => 0,
 				]
 			);
 
@@ -648,13 +654,13 @@ class Index extends ACommandBase {
 
 		$this->runner->setInfoArray(
 			[
-				'errorCurrent'   => $current,
-				'errorTotal'     => $total,
-				'errorMessageA'  => trim($err1),
-				'errorMessageB'  => trim($err2),
-				'errorMessageC'  => trim($err3),
+				'errorCurrent' => $current,
+				'errorTotal' => $total,
+				'errorMessageA' => trim($err1),
+				'errorMessageB' => trim($err2),
+				'errorMessageC' => trim($err3),
 				'errorException' => $this->get('exception', $error, ''),
-				'errorIndex'     => $errorIndex
+				'errorIndex' => $errorIndex
 			]
 		);
 	}
@@ -670,7 +676,7 @@ class Index extends ACommandBase {
 			$this->runner->setInfoArray(
 				[
 					'resultCurrent' => 0,
-					'resultTotal'   => 0,
+					'resultTotal' => 0,
 				]
 			);
 
@@ -704,13 +710,13 @@ class Index extends ACommandBase {
 
 		$this->runner->setInfoArray(
 			[
-				'resultCurrent'  => $current,
-				'resultTotal'    => $total,
+				'resultCurrent' => $current,
+				'resultTotal' => $total,
 				'resultMessageA' => trim($msg1),
 				'resultMessageB' => trim($msg2),
 				'resultMessageC' => trim($msg3),
-				'resultStatus'   => $status,
-				'resultIndex'    => $resultIndex
+				'resultStatus' => $status,
+				'resultIndex' => $resultIndex
 			]
 		);
 		$this->runner->setInfoColored('resultStatus', $type);
@@ -808,10 +814,10 @@ class Index extends ACommandBase {
 		foreach ($indexes as $index) {
 			foreach ($index->getErrors() as $error) {
 				$this->errors[] = [
-					'index'     => $index,
-					'message'   => $error['message'],
+					'index' => $index,
+					'message' => $error['message'],
 					'exception' => $error['exception'],
-					'severity'  => $error['severity']
+					'severity' => $error['severity']
 				];
 			}
 
@@ -830,11 +836,11 @@ class Index extends ACommandBase {
 
 		$this->runner->setInfoArray(
 			[
-				'errorMessageA'  => '',
-				'errorMessageB'  => '',
-				'errorMessageC'  => '',
+				'errorMessageA' => '',
+				'errorMessageB' => '',
+				'errorMessageC' => '',
 				'errorException' => '',
-				'errorIndex'     => ''
+				'errorIndex' => ''
 			]
 		);
 
diff --git a/lib/Command/Migration24.php b/lib/Command/Migration24.php
new file mode 100644
index 0000000..0d9a899
--- /dev/null
+++ b/lib/Command/Migration24.php
@@ -0,0 +1,135 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Command;
+
+
+use OC\Core\Command\Base;
+use OCA\FullTextSearch\Exceptions\DatabaseException;
+use OCA\FullTextSearch\Service\ConfigService;
+use OCA\FullTextSearch\Service\MigrationService;
+use OCP\DB\Exception;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+
+class Migration24 extends Base {
+
+
+	/** @var IDBConnection */
+	private $dbConnection;
+
+	/** @var MigrationService */
+	private $migrationService;
+
+	/** @var ConfigService */
+	private $configService;
+
+
+	/**
+	 * @param IDBConnection $dbConnection
+	 * @param MigrationService $migrationService
+	 * @param ConfigService $configService
+	 */
+	public function __construct(
+		IDBConnection $dbConnection,
+		MigrationService $migrationService,
+		ConfigService $configService
+	) {
+		parent::__construct();
+
+		$this->dbConnection = $dbConnection;
+		$this->migrationService = $migrationService;
+		$this->configService = $configService;
+	}
+
+
+	/**
+	 *
+	 */
+	protected function configure() {
+		parent::configure();
+		$this->setName('fulltextsearch:migration:24')
+			 ->setDescription('Migrate index for NC24');
+	}
+
+
+	/**
+	 * @param InputInterface $input
+	 * @param OutputInterface $output
+	 *
+	 * @return int
+	 * @throws \Exception
+	 */
+	protected function execute(InputInterface $input, OutputInterface $output): int {
+		try {
+			$this->configService->requireMigration24();
+			throw new \Exception('Should not be required');
+		} catch (DatabaseException $e) {
+		}
+
+		$this->runMigration($output);
+
+		return 0;
+	}
+
+
+	private function runMigration(OutputInterface $output): void {
+		$progressBar = new ProgressBar($output, $this->getTotalRows());
+		$this->migrationService->setProgressBar($progressBar);
+		while (true) {
+			if (!$this->migrationService->migrate24Chunk(10000, $progressBar)) {
+				break;
+			}
+		}
+
+		$progressBar->finish();
+		$output->writeln('');
+	}
+
+
+	/**
+	 * @return int
+	 * @throws Exception
+	 */
+	private function getTotalRows(): int {
+		$query = $this->dbConnection->getQueryBuilder();
+		$query->select($query->func()->count('*', 'index_count'))
+			  ->from('fulltextsearch_indexes');
+		$result = $query->executeQuery();
+		$row = $result->fetch();
+		$result->closeCursor();
+
+		return (int)$row['index_count'];
+	}
+
+}
diff --git a/lib/Command/Reset.php b/lib/Command/Reset.php
index c5aa80e..a1e80ab 100644
--- a/lib/Command/Reset.php
+++ b/lib/Command/Reset.php
@@ -87,7 +87,8 @@ class Reset extends ACommandBase {
 		parent::configure();
 		$this->setName('fulltextsearch:reset')
 			 ->setDescription('Reset index')
-			 ->addArgument('provider', InputArgument::OPTIONAL, 'provider');
+			 ->addArgument('provider', InputArgument::OPTIONAL, 'provider id', '')
+			 ->addArgument('collection', InputArgument::OPTIONAL, 'name of the collection', '');
 	}
 
 
@@ -111,7 +112,10 @@ class Reset extends ACommandBase {
 
 		$this->indexService->setRunner($this->runner);
 		try {
-			$this->indexService->resetIndex($this->getProviderIdFromArgument($input));
+			$this->indexService->resetIndex(
+				$input->getArgument('provider'),
+				$input->getArgument('collection')
+			);
 
 		} catch (Exception $e) {
 			throw $e;
@@ -124,21 +128,6 @@ class Reset extends ACommandBase {
 
 
 	/**
-	 * @param InputInterface $input
-	 *
-	 * @return string
-	 */
-	private function getProviderIdFromArgument(InputInterface $input): string {
-		$providerId = $input->getArgument('provider');
-		if ($providerId === null) {
-			$providerId = '';
-		}
-
-		return $providerId;
-	}
-
-
-	/**
 	 * @throws TickDoesNotExistException
 	 */
 	public function abort() {
diff --git a/lib/Controller/CollectionController.php b/lib/Controller/CollectionController.php
new file mode 100644
index 0000000..26f7f1a
--- /dev/null
+++ b/lib/Controller/CollectionController.php
@@ -0,0 +1,164 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2022
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Controller;
+
+
+use OCA\FullTextSearch\AppInfo\Application;
+use OCA\FullTextSearch\Service\CollectionService;
+use OCP\AppFramework\Http\DataResponse;
+use OCP\AppFramework\OCS\OCSException;
+use OCP\AppFramework\OCSController;
+use OCP\FullTextSearch\Model\IIndexDocument;
+use OCP\IRequest;
+
+
+class CollectionController extends OCSController {
+
+
+	/** @var CollectionService */
+	private $collectionService;
+
+
+	/**
+	 * @param IRequest $request
+	 * @param CollectionService $collectionService
+	 */
+	public function __construct(
+		IRequest $request,
+		CollectionService $collectionService
+	) {
+		parent::__construct(Application::APP_ID, $request);
+
+		$this->collectionService = $collectionService;
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param int $length
+	 *
+	 * @return DataResponse
+	 * @throws OCSException
+	 */
+	public function getQueue(string $collection, int $length = 0): DataResponse {
+		try {
+			$this->collectionService->confirmCollection($collection);
+
+			return new DataResponse($this->collectionService->getQueue($collection, $length));
+		} catch (\Exception $e) {
+//			$this->e($e, ['circleId' => $circleId]);
+			throw new OCSException($e->getMessage(), $e->getCode());
+		}
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param string $providerId
+	 * @param string $documentId
+	 *
+	 * @return DataResponse
+	 * @throws OCSException
+	 */
+	public function indexDocument(string $collection, string $providerId, string $documentId): DataResponse {
+		try {
+			$this->collectionService->confirmCollection($collection);
+			$document = $this->collectionService->getDocument(
+				$collection,
+				$providerId,
+				$documentId
+			);
+
+			return new DataResponse($this->displayDocument($document));
+		} catch (\Exception $e) {
+			throw new OCSException($e->getMessage(), $e->getCode());
+		}
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param string $providerId
+	 * @param string $documentId
+	 *
+	 * @return DataResponse
+	 * @throws OCSException
+	 */
+	public function updateStatusDone(
+		string $collection,
+		string $providerId,
+		string $documentId
+	): DataResponse {
+		try {
+			$this->collectionService->confirmCollection($collection);
+			$this->collectionService->setAsDone($collection, $providerId, $documentId);
+
+			return new DataResponse([]);
+		} catch (\Exception $e) {
+			throw new OCSException($e->getMessage(), $e->getCode());
+		}
+	}
+
+
+	/**
+	 * @param IIndexDocument $document
+	 *
+	 * @return array
+	 */
+	private function displayDocument(IIndexDocument $document): array {
+		$display = [
+			'id' => $document->getId(),
+			'providerId' => $document->getProviderId(),
+			'access' => $document->getAccess(),
+			'index' => $document->getIndex(),
+			'title' => $document->getTitle(),
+			'link' => $document->getLink(),
+			'parts' => $document->getParts(),
+			'tags' => $document->getTags(),
+			'metatags' => $document->getMetaTags(),
+			'source' => $document->getSource(),
+			'info' => $document->getInfoAll(),
+			'hash' => $document->getHash(),
+			'modifiedTime' => $document->getModifiedTime()
+		];
+
+		foreach ($document->getMore() as $k => $v) {
+			$display[$k] = $v;
+		}
+
+		$display['content'] = $document->getContent();
+		$display['isContentEncoded'] = $document->isContentEncoded();
+
+		return json_decode(json_encode($display), true);
+	}
+
+}
+
diff --git a/lib/Cron/Index.php b/lib/Cron/Index.php
index 8374583..49374fe 100644
--- a/lib/Cron/Index.php
+++ b/lib/Cron/Index.php
@@ -119,7 +119,7 @@ class Index extends TimedJob {
 		$platform = $wrapper->getPlatform();
 
 		$all = $this->shouldWeGetAllIndex();
-		$indexes = $this->indexService->getQueuedIndexes($all);
+		$indexes = $this->indexService->getQueuedIndexes('', $all);
 
 		foreach ($indexes as $index) {
 			$this->runner->updateAction('indexing');
diff --git a/lib/Cron/Maintenance.php b/lib/Cron/Maintenance.php
new file mode 100644
index 0000000..5a2d505
--- /dev/null
+++ b/lib/Cron/Maintenance.php
@@ -0,0 +1,74 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2022
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Cron;
+
+
+use OC\BackgroundJob\TimedJob;
+use OCA\FullTextSearch\Service\ConfigService;
+use OCA\FullTextSearch\Service\MigrationService;
+
+
+class Maintenance extends TimedJob {
+
+
+	/** @var MigrationService */
+	private $migrationService;
+
+	/** @var ConfigService */
+	private $configService;
+
+
+	/**
+	 *
+	 */
+	public function __construct(MigrationService $migrationService, ConfigService $configService) {
+		$this->setInterval(3600);
+
+		$this->migrationService = $migrationService;
+		$this->configService = $configService;
+	}
+
+
+	/**
+	 * @param mixed $argument
+	 *
+	 * @throws \OCP\DB\Exception
+	 */
+	protected function run($argument) {
+		$size = $this->configService->getAppValue('size_migration_24');
+		if ($size === '') {
+			$size = 10000;
+		}
+
+		$this->migrationService->migrate24Chunk((int)$size);
+	}
+
+}
diff --git a/lib/Db/CoreRequestBuilder.php b/lib/Db/CoreRequestBuilder.php
index d90054c..47018cc 100644
--- a/lib/Db/CoreRequestBuilder.php
+++ b/lib/Db/CoreRequestBuilder.php
@@ -47,7 +47,7 @@ use OCP\IL10N;
  */
 class CoreRequestBuilder {
 
-	const TABLE_INDEXES = 'fulltextsearch_indexes';
+	const TABLE_INDEXES = 'fulltextsearch_index';
 	const TABLE_TICKS = 'fulltextsearch_ticks';
 
 	/** @var IDBConnection */
@@ -130,6 +130,17 @@ class CoreRequestBuilder {
 
 
 	/**
+	 * Limit to the documentId
+	 *
+	 * @param IQueryBuilder $qb
+	 * @param string $collection
+	 */
+	protected function limitToCollection(IQueryBuilder &$qb, string $collection) {
+		$this->limitToDBField($qb, 'collection', $collection);
+	}
+
+
+	/**
 	 * Limit to the entry with at least one Error
 	 *
 	 * @param IQueryBuilder $qb
diff --git a/lib/Db/IndexesRequest.php b/lib/Db/IndexesRequest.php
index f0c7ab9..99745d9 100644
--- a/lib/Db/IndexesRequest.php
+++ b/lib/Db/IndexesRequest.php
@@ -31,9 +31,11 @@ declare(strict_types=1);
 namespace OCA\FullTextSearch\Db;
 
 
+use OCA\FullTextSearch\Exceptions\DatabaseException;
 use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException;
 use OCA\FullTextSearch\Model\Index;
 use OCP\FullTextSearch\Model\IIndex;
+use OCP\IDBConnection;
 
 
 /**
@@ -53,6 +55,7 @@ class IndexesRequest extends IndexesRequestBuilder {
 		$qb = $this->getIndexesInsertSql();
 		$qb->setValue('owner_id', $qb->createNamedParameter($index->getOwnerId()))
 		   ->setValue('provider_id', $qb->createNamedParameter($index->getProviderId()))
+		   ->setValue('collection', $qb->createNamedParameter($index->getCollection()))
 		   ->setValue('document_id', $qb->createNamedParameter($index->getDocumentId()))
 		   ->setValue('source', $qb->createNamedParameter($index->getSource()))
 		   ->setValue('err', $qb->createNamedParameter($index->getErrorCount()))
@@ -73,9 +76,8 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 * @return bool
 	 */
 	public function resetError(Index $index): bool {
-
 		try {
-			$this->getIndex($index->getProviderId(), $index->getDocumentId());
+			$this->getIndex($index->getProviderId(), $index->getDocumentId(), $index->getCollection());
 		} catch (IndexDoesNotExistException $e) {
 			return false;
 		}
@@ -86,7 +88,7 @@ class IndexesRequest extends IndexesRequestBuilder {
 
 		$this->limitToProviderId($qb, $index->getProviderId());
 		$this->limitToDocumentId($qb, $index->getDocumentId());
-
+		$this->limitToCollection($qb, $index->getCollection());
 		$qb->execute();
 
 		return true;
@@ -109,7 +111,6 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 * @return Index[]
 	 */
 	public function getErrorIndexes(): array {
-
 		$qb = $this->getIndexesSelectSql();
 		$this->limitToErr($qb);
 
@@ -129,36 +130,32 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 *
 	 * @return bool
 	 */
-	public function update(Index $index): bool {
-
-		try {
-			$this->getIndex($index->getProviderId(), $index->getDocumentId());
-		} catch (IndexDoesNotExistException $e) {
-			return false;
-		}
-
+	public function update(Index $index, bool $statusOnly = false): bool {
 		$qb = $this->getIndexesUpdateSql();
 		$qb->set('status', $qb->createNamedParameter($index->getStatus()));
-		$qb->set('source', $qb->createNamedParameter($index->getSource()));
-		$qb->set('options', $qb->createNamedParameter(json_encode($index->getOptions())));
 
-		if ($index->getOwnerId() !== '') {
-			$qb->set('owner_id', $qb->createNamedParameter($index->getOwnerId()));
-		}
+		if (!$statusOnly) {
+			$qb->set('source', $qb->createNamedParameter($index->getSource()));
 
-		if ($index->getLastIndex() > 0) {
-			$qb->set('indexed', $qb->createNamedParameter($index->getLastIndex()));
-		}
+			$qb->set('options', $qb->createNamedParameter(json_encode($index->getOptions())));
+
+			if ($index->getOwnerId() !== '') {
+				$qb->set('owner_id', $qb->createNamedParameter($index->getOwnerId()));
+			}
 
-		$qb->set('message', $qb->createNamedParameter(json_encode($index->getErrors())));
-		$qb->set('err', $qb->createNamedParameter($index->getErrorCount()));
+			if ($index->getLastIndex() > 0) {
+				$qb->set('indexed', $qb->createNamedParameter($index->getLastIndex()));
+			}
+
+			$qb->set('message', $qb->createNamedParameter(json_encode($index->getErrors())));
+			$qb->set('err', $qb->createNamedParameter($index->getErrorCount()));
+		}
 
 		$this->limitToProviderId($qb, $index->getProviderId());
 		$this->limitToDocumentId($qb, $index->getDocumentId());
+		$this->limitToCollection($qb, $index->getCollection());
 
-		$qb->execute();
-
-		return true;
+		return ($qb->executeStatement() === 1);
 	}
 
 
@@ -167,27 +164,30 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 * @param string $documentId
 	 * @param int $status
 	 */
-	public function updateStatus(string $providerId, string $documentId, int $status) {
+	public function updateStatus(string $collection, string $providerId, string $documentId, int $status) {
 		$qb = $this->getIndexesUpdateSql();
 		$qb->set('status', $qb->createNamedParameter($status));
 
 		$this->limitToProviderId($qb, $providerId);
 		$this->limitToDocumentId($qb, $documentId);
+		$this->limitToCollection($qb, $collection);
 
 		$qb->execute();
 	}
 
 	/**
+	 * @param string $collection
 	 * @param string $providerId
 	 * @param array $indexes
 	 * @param int $status
 	 */
-	public function updateStatuses(string $providerId, array $indexes, int $status) {
+	public function updateStatuses(string $collection, string $providerId, array $indexes, int $status) {
 		$qb = $this->getIndexesUpdateSql();
 		$qb->set('status', $qb->createNamedParameter($status));
 
 		$this->limitToProviderId($qb, $providerId);
 		$this->limitToDocumentIds($qb, $indexes);
+		$this->limitToCollection($qb, $collection);
 
 		$qb->execute();
 	}
@@ -200,8 +200,20 @@ class IndexesRequest extends IndexesRequestBuilder {
 		$qb = $this->getIndexesDeleteSql();
 		$this->limitToProviderId($qb, $index->getProviderId());
 		$this->limitToDocumentId($qb, $index->getDocumentId());
+		$this->limitToCollection($qb, $index->getCollection());
 
-		$qb->execute();
+		$qb->executeStatement();
+	}
+
+
+	/**
+	 * @param string $collection \
+	 */
+	public function deleteCollection(string $collection): void {
+		$qb = $this->getIndexesDeleteSql();
+		$this->limitToCollection($qb, $collection);
+
+		$qb->executeStatement();
 	}
 
 
@@ -219,10 +231,11 @@ class IndexesRequest extends IndexesRequestBuilder {
 	/**
 	 *
 	 */
-	public function reset() {
+	public function reset(string $collection = ''): void {
 		$qb = $this->getIndexesDeleteSql();
+		$this->limitToCollection($qb, $collection);
 
-		$qb->execute();
+		$qb->executeStatement();
 	}
 
 
@@ -235,10 +248,11 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 * @return Index
 	 * @throws IndexDoesNotExistException
 	 */
-	public function getIndex(string $providerId, string $documentId): Index {
+	public function getIndex(string $providerId, string $documentId, string $collection = ''): Index {
 		$qb = $this->getIndexesSelectSql();
 		$this->limitToProviderId($qb, $providerId);
 		$this->limitToDocumentId($qb, $documentId);
+		$this->limitToCollection($qb, $collection);
 
 		$cursor = $qb->execute();
 		$data = $cursor->fetch();
@@ -256,15 +270,14 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 * return index.
 	 *
 	 * @param string $providerId
-	 * @param array $documentIds
+	 * @param string $documentId
 	 *
 	 * @return Index[]
-	 * @throws IndexDoesNotExistException
 	 */
-	public function getIndexes(string $providerId, array $documentIds): array {
+	public function getIndexes(string $providerId, string $documentId): array {
 		$qb = $this->getIndexesSelectSql();
 		$this->limitToProviderId($qb, $providerId);
-		$this->limitToDocumentIds($qb, $documentIds);
+		$this->limitToDocumentId($qb, $documentId);
 
 		$indexes = [];
 		$cursor = $qb->execute();
@@ -273,10 +286,6 @@ class IndexesRequest extends IndexesRequestBuilder {
 		}
 		$cursor->closeCursor();
 
-		if (sizeof($indexes) === 0) {
-			throw new IndexDoesNotExistException($this->l10n->t('Index not found'));
-		}
-
 		return $indexes;
 	}
 
@@ -286,13 +295,18 @@ class IndexesRequest extends IndexesRequestBuilder {
 	 *
 	 * @return Index[]
 	 */
-	public function getQueuedIndexes(bool $all = false): array {
+	public function getQueuedIndexes(string $collection = '', bool $all = false, int $length = 0): array {
 		$qb = $this->getIndexesSelectSql();
 		$this->limitToQueuedIndexes($qb);
 		if ($all === false) {
 			$this->limitToNoErr($qb);
 		}
 
+		$this->limitToCollection($qb, $collection);
+		if ($length > 0) {
+			$qb->setMaxResults($length);
+		}
+
 		$indexes = [];
 		$cursor = $qb->execute();
 		while ($data = $cursor->fetch()) {
@@ -327,4 +341,69 @@ class IndexesRequest extends IndexesRequestBuilder {
 	}
 
 
+	/**
+	 * @return string[]
+	 */
+	public function getCollections(): array {
+		$qb = $this->dbConnection->getQueryBuilder();
+
+		$qb->select('li.collection')
+		   ->from(self::TABLE_INDEXES, 'li');
+		$qb->andWhere($qb->expr()->nonEmptyString('li.collection'));
+		$qb->groupBy('li.collection');
+
+		$collections = [];
+		$cursor = $qb->executeQuery();
+		while ($data = $cursor->fetch()) {
+			$collections[] = $this->get('collection', $data);
+		}
+		$cursor->closeCursor();
+
+		return array_merge([''], $collections);
+	}
+
+
+	/**
+	 * TODO: remove this in NC30+
+	 *
+	 * @param string $providerId
+	 * @param string $documentId
+	 */
+	public function migrateIndex24(string $providerId, string $documentId): array {
+		try {
+			$this->configService->requireMigration24();
+
+			return [];
+		} catch (DatabaseException $e) {
+		}
+
+		// check data from old table.
+		/** @var IDBConnection $dbConnection */
+		$dbConnection = \OC::$server->get(IDBConnection::class);
+
+		$query = $dbConnection->getQueryBuilder();
+		$query->select('*')
+			  ->from('fulltextsearch_indexes')
+			  ->where($query->expr()->eq('provider_id', $query->createNamedParameter($providerId)))
+			  ->andWhere($query->expr()->eq('document_id', $query->createNamedParameter($documentId)));
+
+		$result = $query->executeQuery();
+		if ($result->rowCount() === 0) {
+			return [];
+		}
+
+		$data = $result->fetch();
+		$index = $this->parseIndexesSelectSql($data);
+		$result->closeCursor();
+
+		$query = $dbConnection->getQueryBuilder();
+		$query->delete('fulltextsearch_indexes')
+			  ->where($query->expr()->eq('provider_id', $query->createNamedParameter($providerId)))
+			  ->andWhere($query->expr()->eq('document_id', $query->createNamedParameter($documentId)));
+		$query->executeStatement();
+
+		$this->create($index);
+
+		return $this->getIndexes($providerId, $documentId);
+	}
 }
diff --git a/lib/Db/IndexesRequestBuilder.php b/lib/Db/IndexesRequestBuilder.php
index d81df09..7634ec6 100644
--- a/lib/Db/IndexesRequestBuilder.php
+++ b/lib/Db/IndexesRequestBuilder.php
@@ -83,8 +83,8 @@ class IndexesRequestBuilder extends CoreRequestBuilder {
 
 		/** @noinspection PhpMethodParametersCountMismatchInspection */
 		$qb->select(
-			'li.owner_id', 'li.provider_id', 'li.document_id', 'li.source', 'li.status',
-			'li.options', 'li.err', 'li.message', 'li.indexed'
+			'li.owner_id', 'li.provider_id', 'li.document_id', 'li.collection', 'li.source',
+			'li.status', 'li.options', 'li.err', 'li.message', 'li.indexed'
 		)
 		   ->from(self::TABLE_INDEXES, 'li');
 
@@ -119,7 +119,8 @@ class IndexesRequestBuilder extends CoreRequestBuilder {
 		$index->setStatus($this->getInt('status', $data))
 			  ->setSource($this->get('source', $data, ''))
 			  ->setOwnerId($this->get('owner_id', $data, ''))
-			  ->setLastIndex($this->getInt('indexed', $data, 0));
+			  ->setLastIndex($this->getInt('indexed', $data, 0))
+			  ->setCollection($this->get('collection', $data));
 		$index->setOptions($this->getArray('options', $data, []));
 		$index->setErrorCount($this->getInt('err', $data, 0));
 		$index->setErrors($this->getArray('message', $data, []));
diff --git a/lib/Exceptions/CollectionArgumentException.php b/lib/Exceptions/CollectionArgumentException.php
new file mode 100644
index 0000000..eb05978
--- /dev/null
+++ b/lib/Exceptions/CollectionArgumentException.php
@@ -0,0 +1,40 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2018
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Exceptions;
+
+
+use Exception;
+
+
+class CollectionArgumentException extends Exception {
+
+}
+
diff --git a/lib/Migration/Version2000Date20201208130255.php b/lib/Migration/Version2000Date20201208130255.php
index 01184dd..dc6d710 100644
--- a/lib/Migration/Version2000Date20201208130255.php
+++ b/lib/Migration/Version2000Date20201208130255.php
@@ -32,46 +32,6 @@ class Version2000Date20201208130255 extends SimpleMigrationStep {
 		/** @var ISchemaWrapper $schema */
 		$schema = $schemaClosure();
 
-		if (!$schema->hasTable('fulltextsearch_indexes')) {
-			$table = $schema->createTable('fulltextsearch_indexes');
-			$table->addColumn('provider_id', 'string', [
-				'notnull' => true,
-				'length' => 255,
-			]);
-			$table->addColumn('document_id', 'string', [
-				'notnull' => true,
-				'length' => 254,
-			]);
-			$table->addColumn('source', 'string', [
-				'notnull' => false,
-				'length' => 64,
-			]);
-			$table->addColumn('owner_id', 'string', [
-				'notnull' => true,
-				'length' => 64,
-			]);
-			$table->addColumn('status', 'smallint', [
-				'notnull' => true,
-				'length' => 1,
-			]);
-			$table->addColumn('options', 'string', [
-				'notnull' => false,
-				'length' => 511,
-			]);
-			$table->addColumn('err', 'smallint', [
-				'notnull' => true,
-				'length' => 1,
-			]);
-			$table->addColumn('message', 'text', [
-				'notnull' => false,
-			]);
-			$table->addColumn('indexed', 'bigint', [
-				'notnull' => false,
-				'length' => 6,
-			]);
-			$table->setPrimaryKey(['provider_id', 'document_id']);
-		}
-
 		if (!$schema->hasTable('fulltextsearch_ticks')) {
 			$table = $schema->createTable('fulltextsearch_ticks');
 			$table->addColumn('id', 'bigint', [
diff --git a/lib/Migration/Version2400Date202201301329.php b/lib/Migration/Version2400Date202201301329.php
new file mode 100644
index 0000000..0290be5
--- /dev/null
+++ b/lib/Migration/Version2400Date202201301329.php
@@ -0,0 +1,162 @@
+<?php
+
+declare(strict_types=1);
+
+namespace OCA\FullTextSearch\Migration;
+
+use Closure;
+use OCA\FullTextSearch\Service\ConfigService;
+use OCP\DB\Exception;
+use OCP\DB\ISchemaWrapper;
+use OCP\IDBConnection;
+use OCP\Migration\IOutput;
+use OCP\Migration\SimpleMigrationStep;
+
+/**
+ * Auto-generated migration step: Please modify to your needs!
+ */
+class Version2400Date202201301329 extends SimpleMigrationStep {
+
+
+	/** @var IDBConnection */
+	private $dbConnection;
+
+	/** @var ConfigService */
+	private $configService;
+
+
+	/**
+	 * @param IDBConnection $dbConnection
+	 * @param ConfigService $configService
+	 */
+	public function __construct(IDBConnection $dbConnection, ConfigService $configService) {
+		$this->dbConnection = $dbConnection;
+		$this->configService = $configService;
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 */
+	public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 *
+	 * @return null|ISchemaWrapper
+	 */
+	public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+
+		if (!$schema->hasTable('fulltextsearch_index')) {
+			$table = $schema->createTable('fulltextsearch_index');
+			$table->addColumn(
+				'provider_id', 'string',
+				[
+					'notnull' => true,
+					'length' => 254,
+				]
+			);
+			$table->addColumn(
+				'document_id', 'string',
+				[
+					'notnull' => true,
+					'length' => 254,
+				]
+			);
+			$table->addColumn(
+				'collection', 'string',
+				[
+					'default' => '',
+					'notnull' => false,
+					'length' => 31
+				]
+			);
+			$table->addColumn(
+				'source', 'string',
+				[
+					'notnull' => false,
+					'length' => 64,
+				]
+			);
+			$table->addColumn(
+				'owner_id', 'string',
+				[
+					'notnull' => true,
+					'length' => 64,
+				]
+			);
+			$table->addColumn(
+				'status', 'smallint',
+				[
+					'notnull' => true,
+					'length' => 1,
+				]
+			);
+			$table->addColumn(
+				'options', 'string',
+				[
+					'notnull' => false,
+					'length' => 511,
+				]
+			);
+			$table->addColumn(
+				'err', 'smallint',
+				[
+					'notnull' => true,
+					'length' => 1,
+				]
+			);
+			$table->addColumn(
+				'message', 'text',
+				[
+					'notnull' => false,
+				]
+			);
+			$table->addColumn(
+				'indexed', 'bigint',
+				[
+					'notnull' => false,
+					'length' => 6,
+					'unsigned' => true
+				]
+			);
+			$table->setPrimaryKey(['provider_id', 'document_id', 'collection']);
+			$table->addIndex(['collection']);
+			$table->addIndex(['collection', 'provider_id', 'document_id', 'status'], 'cpds');
+		}
+
+		return $schema;
+	}
+
+	/**
+	 * @param IOutput $output
+	 * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper`
+	 * @param array $options
+	 *
+	 * @throws Exception
+	 */
+	public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options) {
+		/** @var ISchemaWrapper $schema */
+		$schema = $schemaClosure();
+		if (!$schema->hasTable('fulltextsearch_indexes')) {
+			return;
+		}
+
+		$query = $this->dbConnection->getQueryBuilder();
+		$query->select($query->func()->count('*', 'index_count'))
+			  ->from('fulltextsearch_indexes');
+		$result = $query->executeQuery();
+		$row = $result->fetch();
+		$result->closeCursor();
+
+		if ((int)$row['index_count'] > 0) {
+			$this->configService->setAppValue(ConfigService::MIGRATION_24, '0');
+		}
+	}
+}
diff --git a/lib/Model/Index.php b/lib/Model/Index.php
index a324fff..31d74ec 100644
--- a/lib/Model/Index.php
+++ b/lib/Model/Index.php
@@ -34,6 +34,7 @@ namespace OCA\FullTextSearch\Model;
 use ArtificialOwl\MySmallPhpTools\Traits\TArrayTools;
 use JsonSerializable;
 use OCP\FullTextSearch\Model\IIndex;
+use OCP\IURLGenerator;
 
 
 /**
@@ -54,6 +55,9 @@ class Index implements IIndex, JsonSerializable {
 	private $documentId;
 
 	/** @var string */
+	private $collection;
+
+	/** @var string */
 	private $source = '';
 
 	/** @var string */
@@ -80,10 +84,12 @@ class Index implements IIndex, JsonSerializable {
 	 *
 	 * @param string $providerId
 	 * @param string $documentId
+	 * @param string $collection
 	 */
-	public function __construct(string $providerId, string $documentId) {
+	public function __construct(string $providerId, string $documentId, string $collection = '') {
 		$this->providerId = $providerId;
 		$this->documentId = $documentId;
+		$this->collection = $collection;
 	}
 
 
@@ -141,6 +147,25 @@ class Index implements IIndex, JsonSerializable {
 
 
 	/**
+	 * @param string $collection
+	 *
+	 * @return Index
+	 */
+	public function setCollection(string $collection): self {
+		$this->collection = $collection;
+
+		return $this;
+	}
+
+	/**
+	 * @return string
+	 */
+	public function getCollection(): string {
+		return $this->collection;
+	}
+
+
+	/**
 	 * @param int $status
 	 * @param bool $reset
 	 *
@@ -318,9 +343,9 @@ class Index implements IIndex, JsonSerializable {
 	public function addError(string $message, string $exception = '', int $sev = IIndex::ERROR_SEV_3
 	): IIndex {
 		$this->errors[] = [
-			'message'   => substr($message, 0, 1800),
+			'message' => substr($message, 0, 1800),
 			'exception' => $exception,
-			'severity'  => $sev
+			'severity' => $sev
 		];
 
 		$this->err++;
@@ -353,19 +378,38 @@ class Index implements IIndex, JsonSerializable {
 
 
 	/**
+	 * @param IURLGenerator $urlGenerator
+	 *
+	 * @return array{url: string, status: int}
+	 */
+	public function asSitemap(IURLGenerator $urlGenerator): array {
+		return  [
+			'url' => $urlGenerator->linkToOCSRouteAbsolute('fulltextsearch.Collection.indexDocument',
+														   [
+															   'collection' => $this->getCollection(),
+															   'providerId' => $this->getProviderId(),
+															   'documentId' => $this->getDocumentId()
+														   ]
+			),
+			'status' => $this->getStatus()
+		];
+	}
+
+	/**
 	 * @return array
 	 */
 	public function jsonSerialize(): array {
 		return [
-			'ownerId'    => $this->getOwnerId(),
+			'ownerId' => $this->getOwnerId(),
 			'providerId' => $this->getProviderId(),
-			'source'     => $this->getSource(),
+			'collection' => $this->getCollection(),
+			'source' => $this->getSource(),
 			'documentId' => $this->getDocumentId(),
-			'lastIndex'  => $this->getLastIndex(),
-			'errors'     => $this->getErrors(),
+			'lastIndex' => $this->getLastIndex(),
+			'errors' => $this->getErrors(),
 			'errorCount' => $this->getErrorCount(),
-			'status'     => (int)$this->getStatus(),
-			'options'    => $this->getOptions()
+			'status' => (int)$this->getStatus(),
+			'options' => $this->getOptions()
 		];
 	}
 
diff --git a/lib/Model/Runner.php b/lib/Model/Runner.php
index f09075e..0861861 100644
--- a/lib/Model/Runner.php
+++ b/lib/Model/Runner.php
@@ -211,6 +211,15 @@ class Runner implements IRunner {
 	}
 
 	/**
+	 * @param string $info
+	 * @param int $value
+	 */
+	public function setInfoInt(string $info, int $value): void {
+		$this->info[$info] = $value;
+		$this->infoUpdated();
+	}
+
+	/**
 	 * @param array $data
 	 */
 	public function setInfoArray(array $data) {
@@ -271,6 +280,14 @@ class Runner implements IRunner {
 		return $this->get($info, $this->info, '');
 	}
 
+	/**
+	 * @param string $info
+	 *
+	 * @return int
+	 */
+	public function getInfoInt(string $info): int {
+		return $this->getInt($info, $this->info);
+	}
 
 	/**
 	 * @param array $method
@@ -323,10 +340,10 @@ class Runner implements IRunner {
 	public function newIndexError(IIndex $index, string $message, string $class = '', int $sev = 3
 	) {
 		$error = [
-			'index'     => $index,
-			'message'   => $message,
+			'index' => $index,
+			'message' => $message,
 			'exception' => $class,
-			'severity'  => $sev
+			'severity' => $sev
 		];
 
 		foreach ($this->methodOnIndexError as $method) {
@@ -351,10 +368,10 @@ class Runner implements IRunner {
 	 */
 	public function newIndexResult(IIndex $index, string $message, string $status, int $type) {
 		$result = [
-			'index'   => $index,
+			'index' => $index,
 			'message' => $message,
-			'status'  => $status,
-			'type'    => $type
+			'status' => $status,
+			'type' => $type
 		];
 
 		foreach ($this->methodOnIndexResult as $method) {
diff --git a/lib/Service/CliService.php b/lib/Service/CliService.php
index 6e08977..3b16977 100644
--- a/lib/Service/CliService.php
+++ b/lib/Service/CliService.php
@@ -170,7 +170,7 @@ class CliService {
 		$initVar = $this->runner->getInfoAll();
 		$keys = array_keys($initVar);
 		foreach ($keys as $key) {
-			$this->display->setMessage($initVar[$key], $key);
+			$this->display->setMessage((string)$initVar[$key], (string)$key);
 		}
 
 		$this->display->clear();
@@ -225,7 +225,7 @@ class CliService {
 
 		$keys = array_keys($info);
 		foreach ($keys as $k) {
-			$this->display->setMessage((string)$info[$k], $k);
+			$this->display->setMessage((string)$info[$k], (string)$k);
 		}
 		$this->refreshInfo();
 
diff --git a/lib/Service/CollectionService.php b/lib/Service/CollectionService.php
new file mode 100644
index 0000000..4f18afb
--- /dev/null
+++ b/lib/Service/CollectionService.php
@@ -0,0 +1,285 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2022
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Service;
+
+
+use OCA\FullTextSearch\Db\IndexesRequest;
+use OCA\FullTextSearch\Exceptions\CollectionArgumentException;
+use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException;
+use OCA\FullTextSearch\Exceptions\ProviderDoesNotExistException;
+use OCA\FullTextSearch\Model\Index;
+use OCA\FullTextSearch\Model\IndexOptions;
+use OCA\FullTextSearch\Model\Runner;
+use OCP\FullTextSearch\IFullTextSearchProvider;
+use OCP\FullTextSearch\Model\IIndex;
+use OCP\FullTextSearch\Model\IIndexDocument;
+use OCP\IURLGenerator;
+
+
+class CollectionService {
+
+
+	/** @var IURLGenerator */
+	private $urlGenerator;
+
+	/** @var IndexesRequest */
+	private $indexesRequest;
+
+	/** @var ProviderService */
+	private $providerService;
+
+	/** @var IndexService */
+	private $indexService;
+
+	/** @var ConfigService */
+	private $configService;
+
+
+	/** @var Runner */
+	private $runner;
+
+
+	/**
+	 * @param IURLGenerator $urlGenerator
+	 * @param IndexesRequest $indexesRequest
+	 * @param ProviderService $providerService
+	 * @param IndexService $indexService
+	 * @param ConfigService $configService
+	 */
+	public function __construct(
+		IURLGenerator $urlGenerator,
+		IndexesRequest $indexesRequest,
+		ProviderService $providerService,
+		IndexService $indexService,
+		ConfigService $configService
+	) {
+		$this->urlGenerator = $urlGenerator;
+		$this->indexesRequest = $indexesRequest;
+		$this->providerService = $providerService;
+		$this->indexService = $indexService;
+		$this->configService = $configService;
+	}
+
+
+	public function setRunner(Runner $runner): void {
+		$this->runner = $runner;
+	}
+
+
+	/**
+	 * @param string $collection
+	 */
+	public function deleteCollection(string $collection): void {
+		$this->indexesRequest->deleteCollection($collection);
+	}
+
+
+	/**
+	 * @param string $collection
+	 *
+	 * @return bool
+	 */
+	public function hasCollection(string $collection): bool {
+		foreach ($this->getCollections() as $item) {
+			if ($item === $collection) {
+				return true;
+			}
+		}
+
+		return false;
+	}
+
+	/**
+	 * @return string[]
+	 */
+	public function getCollections(): array {
+		return array_filter($this->indexesRequest->getCollections());
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param int $length
+	 *
+	 * @return array
+	 */
+	public function getQueue(string $collection, int $length = 0): array {
+		if ($length === 0) {
+			$length = $this->configService->getAppValueInt(ConfigService::COLLECTION_INDEXING_LIST);
+		}
+
+		return array_map(
+			function (Index $index): array {
+				return $index->asSitemap($this->urlGenerator);
+			},
+			$this->indexesRequest->getQueuedIndexes($collection, false, $length)
+		);
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param string $providerId
+	 * @param string $documentId
+	 *
+	 * @throws IndexDoesNotExistException
+	 */
+	public function setAsDone(string $collection, string $providerId, string $documentId): void {
+		$index = $this->indexesRequest->getIndex($providerId, $documentId, $collection);
+		$index->setStatus(IIndex::INDEX_DONE);
+		$this->indexService->updateIndex($index);
+	}
+
+
+	/**
+	 * @param IFullTextSearchProvider $provider
+	 * @param string $collection
+	 * @param string $userId
+	 * @param IndexOptions $options
+	 */
+	public function initCollectionIndexes(
+		IFullTextSearchProvider $provider,
+		string $collection,
+		string $userId,
+		IndexOptions $options
+	) {
+		$chunks = $provider->generateChunks($userId);
+		if (empty($chunks)) {
+			$chunks = [$userId];
+		}
+
+		$this->updateRunnerInfo('chunkTotal', (string)sizeof($chunks));
+		$chunkCount = 0;
+		foreach ($chunks as $chunk) {
+			$documents = $provider->generateIndexableDocuments($userId, (string)$chunk);
+			$this->updateRunnerInfoArray(
+				[
+					'chunkCurr' => ++$chunkCount,
+					'documentChunk' => sizeof($documents)
+				]
+			);
+
+			$documentCount = 0;
+			foreach ($documents as $document) {
+				$this->updateRunnerInfo('documentCurr', (string)++$documentCount);
+				$curr = $this->runner->getInfoInt('documentTotal');
+				$this->runner->setInfoInt('documentTotal', ++$curr);
+
+				try {
+					$this->indexesRequest->getIndex(
+						$document->getProviderId(),
+						$document->getId(),
+						$collection
+					);
+				} catch (IndexDoesNotExistException $e) {
+					$index = new Index($document->getProviderId(), $document->getId(), $collection);
+					$index->setStatus(IIndex::INDEX_FULL);
+					$index->setOwnerId($document->getAccess()->getOwnerId());
+					$index->setSource($document->getSource());
+					$index->setLastIndex();
+					$this->indexesRequest->create($index);
+
+					$curr = $this->runner->getInfoInt('indexCount');
+					$this->runner->setInfoInt('indexCount', ++$curr);
+				}
+			}
+		}
+	}
+
+
+	/**
+	 * @param string $collection
+	 *
+	 * @throws CollectionArgumentException
+	 */
+	public function confirmCollectionString(string $collection): void {
+//		throw new CollectionArgumentException();
+	}
+
+	/**
+	 * @param string $collection
+	 *
+	 * @throws CollectionArgumentException
+	 */
+	public function confirmCollection(string $collection): void {
+		$this->confirmCollectionString($collection);
+
+		if (!$this->hasCollection($collection)) {
+			throw new CollectionArgumentException('collection does not exist');
+		}
+	}
+
+
+	/**
+	 * @param string $collection
+	 * @param string $providerId
+	 * @param string $documentId
+	 *
+	 * @return IIndexDocument
+	 * @throws ProviderDoesNotExistException
+	 * @throws IndexDoesNotExistException
+	 */
+	public function getDocument(string $collection, string $providerId, string $documentId): IIndexDocument {
+		$wrapped = $this->providerService->getProvider($providerId);
+		$provider = $wrapped->getProvider();
+
+		$index = $this->indexService->getIndex($providerId, $documentId, $collection);
+		$index->setStatus(IIndex::INDEX_FULL);
+
+		return $provider->updateDocument($index);
+	}
+
+
+	/**
+	 * @param string $info
+	 * @param string $value
+	 */
+	private function updateRunnerInfo(string $info, string $value): void {
+		if (is_null($this->runner)) {
+			return;
+		}
+
+		$this->runner->setInfo($info, $value);
+	}
+
+
+	/**
+	 * @param array $data
+	 */
+	private function updateRunnerInfoArray(array $data): void {
+		if (is_null($this->runner)) {
+			return;
+		}
+
+		$this->runner->setInfoArray($data);
+	}
+
+}
diff --git a/lib/Service/ConfigService.php b/lib/Service/ConfigService.php
index 457ec1a..1bd5a2e 100644
--- a/lib/Service/ConfigService.php
+++ b/lib/Service/ConfigService.php
@@ -32,6 +32,7 @@ namespace OCA\FullTextSearch\Service;
 
 
 use OCA\FullTextSearch\AppInfo\Application;
+use OCA\FullTextSearch\Exceptions\DatabaseException;
 use OCA\FullTextSearch\Exceptions\ProviderOptionsDoesNotExistException;
 use OCP\IConfig;
 use OCP\PreConditionNotMetException;
@@ -51,14 +52,21 @@ class ConfigService {
 	const PROVIDER_INDEXED = 'provider_indexed';
 	const CRON_LAST_ERR_RESET = 'cron_err_reset';
 	const TICK_TTL = 'tick_ttl';
+	const COLLECTION_INDEXING_LIST = 'collection_indexing_list';
+
+
+	// Temp. can be removed after few major releases
+	const MIGRATION_24 = 'migration_24';
 
 	/** @var array */
 	public $defaults = [
-		self::SEARCH_PLATFORM     => '',
-		self::APP_NAVIGATION      => '0',
-		self::PROVIDER_INDEXED    => '',
+		self::SEARCH_PLATFORM => '',
+		self::APP_NAVIGATION => '0',
+		self::PROVIDER_INDEXED => '',
 		self::CRON_LAST_ERR_RESET => '0',
-		self::TICK_TTL            => '1800'
+		self::TICK_TTL => '1800',
+		self::COLLECTION_INDEXING_LIST => 50,
+		self::MIGRATION_24 => 1
 	];
 
 
@@ -135,15 +143,25 @@ class ConfigService {
 	 * @return string
 	 */
 	public function getAppValue(string $key): string {
-		$defaultValue = null;
+		$defaultValue = '';
 		if (array_key_exists($key, $this->defaults)) {
 			$defaultValue = $this->defaults[$key];
 		}
 
-		return $this->config->getAppValue(Application::APP_ID, $key, $defaultValue);
+		return (string)$this->config->getAppValue(Application::APP_ID, $key, $defaultValue);
 	}
 
 	/**
+	 * @param string $config
+	 *
+	 * @return int
+	 */
+	public function getAppValueInt(string $config): int {
+		return (int)$this->getAppValue($config);
+	}
+
+
+	/**
 	 * Set a value by key
 	 *
 	 * @param string $key
@@ -274,4 +292,16 @@ class ConfigService {
 
 		return $ver[0];
 	}
+
+
+	/**
+	 * @throws DatabaseException
+	 */
+	public function requireMigration24(): void {
+		if ($this->getAppValueInt(self::MIGRATION_24) === 1) {
+			return;
+		}
+
+		throw new DatabaseException('please run ./occ fulltextsearch:migration:24');
+	}
 }
diff --git a/lib/Service/IndexService.php b/lib/Service/IndexService.php
index 2192536..b30283b 100644
--- a/lib/Service/IndexService.php
+++ b/lib/Service/IndexService.php
@@ -33,9 +33,9 @@ namespace OCA\FullTextSearch\Service;
 
 use Exception;
 use OCA\FullTextSearch\Db\IndexesRequest;
-use OCA\FullTextSearch\Exceptions\DatabaseException;
 use OCA\FullTextSearch\Exceptions\IndexDoesNotExistException;
 use OCA\FullTextSearch\Exceptions\NotIndexableDocumentException;
+use OCA\FullTextSearch\Exceptions\ProviderDoesNotExistException;
 use OCA\FullTextSearch\Model\Index;
 use OCA\FullTextSearch\Model\IndexOptions;
 use OCA\FullTextSearch\Model\Runner;
@@ -59,18 +59,12 @@ class IndexService implements IIndexService {
 	/** @var IndexesRequest */
 	private $indexesRequest;
 
-	/** @var ConfigService */
-	private $configService;
-
 	/** @var ProviderService */
 	private $providerService;
 
 	/** @var PlatformService */
 	private $platformService;
 
-	/** @var MiscService */
-	private $miscService;
-
 
 	/** @var Runner */
 	private $runner = null;
@@ -86,20 +80,17 @@ class IndexService implements IIndexService {
 	 * IndexService constructor.
 	 *
 	 * @param IndexesRequest $indexesRequest
-	 * @param ConfigService $configService
 	 * @param ProviderService $providerService
 	 * @param PlatformService $platformService
-	 * @param MiscService $miscService
 	 */
 	public function __construct(
-		IndexesRequest $indexesRequest, ConfigService $configService,
-		ProviderService $providerService, PlatformService $platformService, MiscService $miscService
+		IndexesRequest $indexesRequest,
+		ProviderService $providerService,
+		PlatformService $platformService
 	) {
 		$this->indexesRequest = $indexesRequest;
-		$this->configService = $configService;
 		$this->providerService = $providerService;
 		$this->platformService = $platformService;
-		$this->miscService = $miscService;
 	}
 
 
@@ -167,15 +158,15 @@ class IndexService implements IIndexService {
 		$this->updateRunnerAction('generateIndex' . $provider->getName());
 		$this->updateRunnerInfoArray(
 			[
-				'userId'          => $userId,
-				'providerId'      => $provider->getId(),
-				'providerName'    => $provider->getName(),
-				'chunkCurrent'    => 0,
-				'chunkTotal'      => 0,
+				'userId' => $userId,
+				'providerId' => $provider->getId(),
+				'providerName' => $provider->getName(),
+				'chunkCurrent' => 0,
+				'chunkTotal' => 0,
 				'documentCurrent' => 0,
-				'documentTotal'   => 0,
-				'info'            => '',
-				'title'           => ''
+				'documentTotal' => 0,
+				'info' => '',
+				'title' => ''
 			]
 		);
 
@@ -193,14 +184,14 @@ class IndexService implements IIndexService {
 			$this->currentTotalDocuments = sizeof($documents);
 			$this->updateRunnerInfoArray(
 				[
-					'documentTotal'   => $this->currentTotalDocuments,
+					'documentTotal' => $this->currentTotalDocuments,
 					'documentCurrent' => 0
 				]
 			);
 
 			//$maxSize = sizeof($documents);
 
-			$toIndex = $this->updateDocumentsWithCurrIndex($provider, $documents, $options);
+			$toIndex = $this->updateDocumentsWithCurrIndex($provider, '', $documents, $options);
 			$this->indexDocuments($platform, $provider, $toIndex, $options);
 		}
 	}
@@ -214,14 +205,16 @@ class IndexService implements IIndexService {
 	 * @return IIndexDocument[]
 	 * @throws Exception
 	 */
-	private function updateDocumentsWithCurrIndex(
-		IFullTextSearchProvider $provider, array $documents, IIndexOptions $options
+	public function updateDocumentsWithCurrIndex(
+		IFullTextSearchProvider $provider,
+		string $collection,
+		array $documents,
+		IIndexOptions $options
 	): array {
 
 		$result = [];
 		$count = 0;
 		foreach ($documents as $document) {
-
 			if ($count % 1000 === 0) {
 				$this->updateRunnerAction('compareWithCurrentIndex', true);
 				$this->updateRunnerInfo('documentCurrent', (string)$count);
@@ -229,10 +222,13 @@ class IndexService implements IIndexService {
 			$count++;
 
 			try {
-				$index =
-					$this->indexesRequest->getIndex($document->getProviderId(), $document->getId());
+				$index = $this->indexesRequest->getIndex(
+					$document->getProviderId(),
+					$document->getId(),
+					$collection
+				);
 			} catch (IndexDoesNotExistException $e) {
-				$index = new Index($document->getProviderId(), $document->getId());
+				$index = new Index($document->getProviderId(), $document->getId(), $collection);
 				$index->setStatus(Index::INDEX_FULL);
 				$index->setLastIndex();
 			}
@@ -267,7 +263,9 @@ class IndexService implements IIndexService {
 	 *
 	 * @return bool
 	 */
-	private function isDocumentUpToDate(IFullTextSearchProvider $provider, IIndexDocument $document
+	private function isDocumentUpToDate(
+		IFullTextSearchProvider $provider,
+		IIndexDocument $document
 	): bool {
 		$index = $document->getIndex();
 		if (!$index->isStatus(Index::INDEX_OK)) {
@@ -305,11 +303,11 @@ class IndexService implements IIndexService {
 				$this->updateRunnerAction('fillDocument', true);
 				$this->updateRunnerInfoArray(
 					[
-						'documentId'    => $document->getId(),
-						'info'          => '',
-						'title'         => '',
-						'content'       => '',
-						'status'        => '',
+						'documentId' => $document->getId(),
+						'info' => '',
+						'title' => '',
+						'content' => '',
+						'status' => '',
 						'statusColored' => ''
 					]
 				);
@@ -317,7 +315,7 @@ class IndexService implements IIndexService {
 				$provider->fillIndexDocument($document);
 				$this->updateRunnerInfoArray(
 					[
-						'title'   => $document->getTitle(),
+						'title' => $document->getTitle(),
 						'content' => $document->getContentSize()
 					]
 				);
@@ -366,8 +364,8 @@ class IndexService implements IIndexService {
 		$this->updateRunnerInfoArray(
 			[
 				'documentId' => $document->getId(),
-				'title'      => $document->getTitle(),
-				'content'    => $document->getContentSize()
+				'title' => $document->getTitle(),
+				'content' => $document->getContentSize()
 			]
 		);
 
@@ -394,7 +392,7 @@ class IndexService implements IIndexService {
 		$this->updateRunnerInfoArray(
 			[
 				'providerName' => $provider->getName(),
-				'userId'       => $index->getOwnerId(),
+				'userId' => $index->getOwnerId(),
 			]
 		);
 
@@ -424,8 +422,8 @@ class IndexService implements IIndexService {
 		$this->updateRunnerInfoArray(
 			[
 				'documentId' => $document->getId(),
-				'title'      => $document->getTitle(),
-				'content'    => $document->getContentSize()
+				'title' => $document->getTitle(),
+				'content' => $document->getContentSize()
 			]
 		);
 
@@ -438,28 +436,19 @@ class IndexService implements IIndexService {
 
 	/**
 	 * @param Index[] $indexes
-	 *
-	 * @throws DatabaseException
 	 */
 	public function updateIndexes(array $indexes) {
-		try {
-			foreach ($indexes as $index) {
-				$this->updateIndex($index);
-			}
-			$this->resetErrorFromQueue();
-		} catch (Exception $e) {
-			throw new DatabaseException($e->getMessage());
+		foreach ($indexes as $index) {
+			$this->updateIndex($index);
 		}
+		$this->resetErrorFromQueue();
 	}
 
 
 	/**
 	 * @param IIndex $index
-	 *
-	 * @throws Exception
 	 */
 	public function updateIndex(IIndex $index) {
-
 		/** @var Index $index */
 		$this->updateIndexError($index);
 		if ($index->isStatus(IIndex::INDEX_REMOVE)) {
@@ -502,22 +491,20 @@ class IndexService implements IIndexService {
 	 * @throws Exception
 	 */
 	public function updateIndexStatus(
-		string $providerId, string $documentId, int $status, bool $reset = false
+		string $providerId,
+		string $documentId,
+		int $status,
+		bool $reset = false
 	) {
-		if ($reset === true) {
-			$this->indexesRequest->updateStatus($providerId, $documentId, $status);
-
-			return;
+		$indexes = $this->indexesRequest->getIndexes($providerId, $documentId);
+		if (empty($indexes)) {
+			$indexes = $this->indexesRequest->migrateIndex24($providerId, $documentId);
 		}
 
-		try {
-			$curr = $this->getIndex($providerId, $documentId);
-		} catch (IndexDoesNotExistException $e) {
-			return;
+		foreach ($indexes as $index) {
+			$index->setStatus($status);
+			$this->updateIndex($index);
 		}
-
-		$curr->setStatus($status);
-		$this->updateIndex($curr);
 	}
 
 
@@ -526,27 +513,18 @@ class IndexService implements IIndexService {
 	 * @param array $documentIds
 	 * @param int $status
 	 * @param bool $reset
-	 *
-	 * @throws DatabaseException
 	 */
 	public function updateIndexesStatus(
-		string $providerId, array $documentIds, int $status, bool $reset = false
+		string $providerId,
+		array $documentIds,
+		int $status,
+		bool $reset = false
 	) {
-		if ($reset === true) {
-			$this->indexesRequest->updateStatuses($providerId, $documentIds, $status);
-
-			return;
-		}
-
-		try {
-			$all = $this->getIndexes($providerId, $documentIds);
-		} catch (IndexDoesNotExistException $e) {
-			return;
-		}
-
-		foreach ($all as $curr) {
-			$curr->setStatus($status);
-			$this->updateIndexes([$curr]);
+		foreach ($documentIds as $documentId) {
+			try {
+				$this->updateIndexStatus($providerId, $documentId, $status, $reset);
+			} catch (Exception $e) {
+			}
 		}
 	}
 
@@ -588,39 +566,41 @@ class IndexService implements IIndexService {
 
 	/**
 	 * @param string $providerId
-	 * @param array $documentId
+	 * @param string $documentId
 	 *
 	 * @return Index[]
-	 * @throws IndexDoesNotExistException
 	 */
-	public function getIndexes($providerId, $documentId) {
+	public function getIndexes(string $providerId, string $documentId): array {
 		return $this->indexesRequest->getIndexes($providerId, $documentId);
 	}
 
 
 	/**
+	 * @param string $collection
 	 * @param bool $all
+	 * @param int $length
 	 *
 	 * @return Index[]
 	 */
-	public function getQueuedIndexes(bool $all = false): array {
-		return $this->indexesRequest->getQueuedIndexes($all);
+	public function getQueuedIndexes(string $collection = '', bool $all = false, int $length = -1): array {
+		return $this->indexesRequest->getQueuedIndexes($collection, $all, $length);
 	}
 
 
 	/**
 	 * @param string $providerId
+	 * @param string $collection
 	 *
-	 * @throws Exception
+	 * @throws ProviderDoesNotExistException
 	 */
-	public function resetIndex(string $providerId = '') {
+	public function resetIndex(string $providerId = '', string $collection = '') {
 		$wrapper = $this->platformService->getPlatform();
 		$platform = $wrapper->getPlatform();
 
 		if ($providerId === '') {
 			$platform->resetIndex('all');
 			$this->providerService->setProvidersAsNotIndexed();
-			$this->indexesRequest->reset();
+			$this->indexesRequest->reset($collection);
 
 			return;
 		} else {
@@ -628,13 +608,13 @@ class IndexService implements IIndexService {
 			$providers = [$providerWrapper->getProvider()];
 		}
 
-		foreach ($providers AS $provider) {
+		foreach ($providers as $provider) {
 			// TODO: need to specify the map to remove
 			// TODO: need to remove entries with type=providerId
 //			$provider->onResettingIndex($platform);
 
 			$platform->resetIndex($provider->getId());
-			$this->providerService->setProviderAsIndexed($provider, false);
+			$this->providerService->setProviderAsIndexed($provider->getId(), false);
 			$this->indexesRequest->deleteFromProviderId($provider->getId());
 		}
 	}
@@ -647,8 +627,8 @@ class IndexService implements IIndexService {
 	 * @return IIndex
 	 * @throws IndexDoesNotExistException
 	 */
-	public function getIndex(string $providerId, string $documentId): IIndex {
-		return $this->indexesRequest->getIndex($providerId, $documentId);
+	public function getIndex(string $providerId, string $documentId, string $collection = ''): IIndex {
+		return $this->indexesRequest->getIndex($providerId, $documentId, $collection);
 	}
 
 
@@ -659,23 +639,33 @@ class IndexService implements IIndexService {
 	 * @param int $status
 	 *
 	 * @return IIndex
+	 * @throws IndexDoesNotExistException
 	 */
-	public function createIndex(string $providerId, string $documentId, string $userId, int $status
+	public function createIndex(
+		string $providerId,
+		string $documentId,
+		string $userId,
+		int $status
 	): IIndex {
+		$index = null;
+		foreach ($this->indexesRequest->getCollections() as $collection) {
+			try {
+				$index = $this->indexesRequest->getIndex($providerId, $documentId, $collection);
+				$index->setStatus($status, true);
+				$this->indexesRequest->update($index, true);
+			} catch (IndexDoesNotExistException $e) {
+				$index = new Index($providerId, $documentId, $collection);
+				$index->setOwnerId($userId);
+				$index->setStatus($status);
+				$this->indexesRequest->create($index);
+			}
+		}
 
-		try {
-			$known = $this->indexesRequest->getIndex($providerId, $documentId);
-			$known->setStatus($status);
-			$this->indexesRequest->updateStatus($providerId, $documentId, $status);
-
-			return $known;
-		} catch (IndexDoesNotExistException $e) {
+		if (is_null($index)) {
+			throw new IndexDoesNotExistException();
 		}
 
-		$index = new Index($providerId, $documentId);
-		$index->setOwnerId($userId);
-		$index->setStatus($status);
-		$this->indexesRequest->create($index);
+		$index->setCollection('');
 
 		return $index;
 	}
diff --git a/lib/Service/MigrationService.php b/lib/Service/MigrationService.php
new file mode 100644
index 0000000..4b3c793
--- /dev/null
+++ b/lib/Service/MigrationService.php
@@ -0,0 +1,182 @@
+<?php
+declare(strict_types=1);
+
+
+/**
+ * FullTextSearch - Full text search framework for Nextcloud
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2022
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\FullTextSearch\Service;
+
+
+use OCP\DB\Exception;
+use OCP\DB\IResult;
+use OCP\DB\QueryBuilder\IQueryBuilder;
+use OCP\IDBConnection;
+use Symfony\Component\Console\Helper\ProgressBar;
+
+
+class MigrationService {
+
+
+	/** @var IDBConnection */
+	private $dbConnection;
+
+	/** @var ConfigService */
+	private $configService;
+
+
+	/** @var ProgressBar */
+	private $progressBar;
+
+
+	/**
+	 * @param IDBConnection $dbConnection
+	 * @param ConfigService $configService
+	 */
+	public function __construct(
+		IDBConnection $dbConnection,
+		ConfigService $configService
+	) {
+		$this->dbConnection = $dbConnection;
+		$this->configService = $configService;
+	}
+
+
+	public function setProgressBar(ProgressBar $progressBar): void {
+		$this->progressBar = $progressBar;
+	}
+
+
+	/**
+	 * @param int $size
+	 *
+	 * @return bool
+	 * @throws Exception
+	 */
+	public function migrate24Chunk(int $size = 10000): bool {
+		if ($this->configService->getAppValueInt(ConfigService::MIGRATION_24) === 1) {
+			return false;
+		}
+
+		if ($size === 0) {
+			return false;
+		}
+
+		$oldData = $this->getChunkOldData($size);
+		if ($oldData->rowCount() === 0) {
+			$this->configService->setAppValue(ConfigService::MIGRATION_24, '1');
+			if ($this->dbConnection->tableExists('fulltextsearch_indexes')) {
+				$this->dbConnection->dropTable('fulltextsearch_indexes');
+			}
+
+			return false;
+		}
+
+		$this->createNewEntries($oldData);
+
+		return true;
+	}
+
+
+	/**
+	 * @param int $size
+	 *
+	 * @return IResult
+	 * @throws Exception
+	 */
+	private function getChunkOldData(int $size): IResult {
+		$query = $this->dbConnection->getQueryBuilder();
+		$query->select('*')
+			  ->from('fulltextsearch_indexes')
+			  ->orderBy('provider_id', 'asc')
+			  ->addOrderBy('document_id', 'asc')
+			  ->setFirstResult(0)
+			  ->setMaxResults($size);
+
+		return $query->executeQuery();
+	}
+
+
+	/**
+	 * @param IResult $oldData
+	 *
+	 * @throws Exception
+	 */
+	private function createNewEntries(IResult $oldData): void {
+		$create = $this->dbConnection->getQueryBuilder();
+		$create->insert('fulltextsearch_index')
+			   ->values(
+				   [
+					   'owner_id' => $create->createParameter('owner_id'),
+					   'provider_id' => $create->createParameter('provider_id'),
+					   'collection' => $create->createParameter('collection'),
+					   'document_id' => $create->createParameter('document_id'),
+					   'source' => $create->createParameter('source'),
+					   'err' => $create->createParameter('err'),
+					   'message' => $create->createParameter('message'),
+					   'status' => $create->createParameter('status'),
+					   'options' => $create->createParameter('options'),
+					   'indexed' => $create->createParameter('indexed')
+				   ]
+			   );
+
+		while ($row = $oldData->fetch()) {
+			$create->setParameter('owner_id', $row['owner_id'])
+				   ->setParameter('provider_id', $row['provider_id'])
+				   ->setParameter('collection', '')
+				   ->setParameter('document_id', $row['document_id'])
+				   ->setParameter('source', $row['source'])
+				   ->setParameter('err', $row['err'], IQueryBuilder::PARAM_INT)
+				   ->setParameter('message', $row['message'])
+				   ->setParameter('status', $row['status'], IQueryBuilder::PARAM_INT)
+				   ->setParameter('options', $row['options'])
+				   ->setParameter('indexed', $row['indexed'], IQueryBuilder::PARAM_INT);
+
+			if (!is_null($this->progressBar)) {
+				$this->progressBar->advance();
+			}
+
+			try {
+				$create->executeStatement();
+			} catch (Exception $e) {
+				if ($e->getReason() !== Exception::REASON_UNIQUE_CONSTRAINT_VIOLATION) {
+					throw $e;
+				}
+			}
+
+			$delete = $this->dbConnection->getQueryBuilder();
+			$delete->delete('fulltextsearch_indexes');
+			$delete->where(
+				$delete->expr()->eq('provider_id', $delete->createNamedParameter($row['provider_id']))
+			);
+			$delete->andWhere(
+				$delete->expr()->eq('document_id', $delete->createNamedParameter($row['document_id']))
+			);
+
+			$delete->executeStatement();
+		}
+	}
+}
diff --git a/lib/Service/ProviderService.php b/lib/Service/ProviderService.php
index 17cda2d..093cfb4 100644
--- a/lib/Service/ProviderService.php
+++ b/lib/Service/ProviderService.php
@@ -244,12 +244,13 @@ class ProviderService implements IProviderService {
 
 
 	/**
-	 * @param IFullTextSearchProvider $provider
+	 * @param string $providerId
 	 * @param bool $boolean
 	 */
-	public function setProviderAsIndexed(IFullTextSearchProvider $provider, bool $boolean) {
+	public function setProviderAsIndexed(string $providerId, bool $boolean) {
 		$this->configService->setProviderOptions(
-			$provider->getId(), ConfigService::PROVIDER_INDEXED, (($boolean) ? '1' : '0')
+			$providerId,
+			ConfigService::PROVIDER_INDEXED, (($boolean) ? '1' : '0')
 		);
 	}
 
@@ -291,7 +292,7 @@ class ProviderService implements IProviderService {
 		if (array_key_exists('@attributes', $providers)) {
 			$providers = [$providers];
 		}
-		foreach ($providers AS $provider) {
+		foreach ($providers as $provider) {
 			if (is_array($provider)) {
 				$attributes = $provider['@attributes'];
 				if (array_key_exists('min-version', $attributes)
@@ -326,7 +327,7 @@ class ProviderService implements IProviderService {
 	 * @throws Exception
 	 */
 	private function providerIdMustBeUnique(IFullTextSearchProvider $provider) {
-		foreach ($this->providers AS $providerWrapper) {
+		foreach ($this->providers as $providerWrapper) {
 			$knownProvider = $providerWrapper->getProvider();
 			if ($knownProvider->getId() === $provider->getId()) {
 				throw new ProviderIsNotUniqueException(
@@ -346,7 +347,7 @@ class ProviderService implements IProviderService {
 		$arr = [];
 		foreach ($providers as $provider) {
 			$arr[] = [
-				'id'   => $provider->getId(),
+				'id' => $provider->getId(),
 				'name' => $provider->getName()
 			];
 		}