Fix sync'ing with many known UIDs

When looking for changed and vanished UIDs we typically have to send the
list of the known UIDs to the IMAP server to know what went missing
meanwhile. With the number of UIDs sent, the size of the IMAP command
grows. At some point the server will just refuse the command due to its
size.
To circumvent this scenario, this patch splits the sync process into
new, changed and vanished. New is cheap in terms of data sent, so it
stays untouched. Changed and vanished will now split the known UIDs list
into chunks and retrieve partial sync results that way.

Signed-off-by: Christoph Wurst <christoph@winzerhof-wurst.at>

1 files changed, 84 insertions, 9 deletions

diff --git a/lib/IMAP/Sync/Synchronizer.php b/lib/IMAP/Sync/Synchronizer.php
index c0cdbd7c1..ef1dcdf13 100644
--- a/lib/IMAP/Sync/Synchronizer.php
+++ b/lib/IMAP/Sync/Synchronizer.php
@@ -31,8 +31,11 @@ use Horde_Imap_Client_Ids;
 use Horde_Imap_Client_Mailbox;
 use OCA\Mail\Exception\UidValidityChangedException;
 use OCA\Mail\IMAP\MessageMapper;
+use function array_chunk;
+use function array_merge;
 
 class Synchronizer {
+	private const UID_CHUNK_SIZE = 15000;
 
 	/** @var MessageMapper */
 	private $messageMapper;
@@ -53,14 +56,24 @@ class Synchronizer {
 	 */
 	public function sync(Horde_Imap_Client_Base $imapClient,
 						 Request $request,
-						 int $criteria = Horde_Imap_Client::SYNC_NEWMSGSUIDS|Horde_Imap_Client::SYNC_FLAGSUIDS|Horde_Imap_Client::SYNC_VANISHEDUIDS): Response {
+						 int $criteria = Horde_Imap_Client::SYNC_NEWMSGSUIDS | Horde_Imap_Client::SYNC_FLAGSUIDS | Horde_Imap_Client::SYNC_VANISHEDUIDS): Response {
 		$mailbox = new Horde_Imap_Client_Mailbox($request->getMailbox());
-		$ids = new Horde_Imap_Client_Ids($request->getUids());
 		try {
-			$hordeSync = $imapClient->sync($mailbox, $request->getToken(), [
-				'criteria' => $criteria,
-				'ids' => $ids
-			]);
+			if ($criteria & Horde_Imap_Client::SYNC_NEWMSGS) {
+				$newUids = $this->getNewMessageUids($imapClient, $mailbox, $request);
+			} else {
+				$newUids = [];
+			}
+			if ($criteria & Horde_Imap_Client::SYNC_FLAGSUIDS) {
+				$changedUids = $this->getChangedMessageUids($imapClient, $mailbox, $request);
+			} else {
+				$changedUids = [];
+			}
+			if ($criteria & Horde_Imap_Client::SYNC_VANISHEDUIDS) {
+				$vanishedUids = $this->getVanishedMessageUids($imapClient, $mailbox, $request);
+			} else {
+				$vanishedUids = [];
+			}
 		} catch (Horde_Imap_Client_Exception_Sync $e) {
 			if ($e->getCode() === Horde_Imap_Client_Exception_Sync::UIDVALIDITY_CHANGED) {
 				throw new UidValidityChangedException();
@@ -68,10 +81,72 @@ class Synchronizer {
 			throw $e;
 		}
 
-		$newMessages = $this->messageMapper->findByIds($imapClient, $request->getMailbox(), $hordeSync->newmsgsuids->ids);
-		$changedMessages = $this->messageMapper->findByIds($imapClient, $request->getMailbox(), $hordeSync->flagsuids->ids);
-		$vanishedMessageUids = $hordeSync->vanisheduids->ids;
+		$newMessages = $this->messageMapper->findByIds($imapClient, $request->getMailbox(), $newUids);
+		$changedMessages = $this->messageMapper->findByIds($imapClient, $request->getMailbox(), $changedUids);
+		$vanishedMessageUids = $vanishedUids;
 
 		return new Response($newMessages, $changedMessages, $vanishedMessageUids);
 	}
+
+	/**
+	 * @param Horde_Imap_Client_Base $imapClient
+	 * @param Horde_Imap_Client_Mailbox $mailbox
+	 * @param Request $request
+	 *
+	 * @return array
+	 * @throws Horde_Imap_Client_Exception
+	 * @throws Horde_Imap_Client_Exception_Sync
+	 */
+	private function getNewMessageUids(Horde_Imap_Client_Base $imapClient, Horde_Imap_Client_Mailbox $mailbox, Request $request): array {
+		$newUids = $imapClient->sync($mailbox, $request->getToken(), [
+			'criteria' => Horde_Imap_Client::SYNC_NEWMSGS,
+		])->newmsgsuids->ids;
+		return $newUids;
+	}
+
+	/**
+	 * @param Horde_Imap_Client_Base $imapClient
+	 * @param Horde_Imap_Client_Mailbox $mailbox
+	 * @param Request $request
+	 *
+	 * @return array
+	 */
+	private function getChangedMessageUids(Horde_Imap_Client_Base $imapClient, Horde_Imap_Client_Mailbox $mailbox, Request $request): array {
+		$changedUids = array_merge(
+			[], // for php<7.4 https://www.php.net/manual/en/function.array-merge.php
+			...array_map(
+				function (array $uids) use ($imapClient, $mailbox, $request) {
+					return $imapClient->sync($mailbox, $request->getToken(), [
+						'criteria' => Horde_Imap_Client::SYNC_FLAGSUIDS,
+						'ids' => new Horde_Imap_Client_Ids($uids),
+					])->flagsuids->ids;
+				},
+				array_chunk($request->getUids(), self::UID_CHUNK_SIZE)
+			)
+		);
+		return $changedUids;
+	}
+
+	/**
+	 * @param Horde_Imap_Client_Base $imapClient
+	 * @param Horde_Imap_Client_Mailbox $mailbox
+	 * @param Request $request
+	 *
+	 * @return array
+	 */
+	private function getVanishedMessageUids(Horde_Imap_Client_Base $imapClient, Horde_Imap_Client_Mailbox $mailbox, Request $request): array {
+		$vanishedUids = array_merge(
+			[], // for php<7.4 https://www.php.net/manual/en/function.array-merge.php
+			...array_map(
+				function (array $uids) use ($imapClient, $mailbox, $request) {
+					return $imapClient->sync($mailbox, $request->getToken(), [
+						'criteria' => Horde_Imap_Client::SYNC_VANISHEDUIDS,
+						'ids' => new Horde_Imap_Client_Ids($uids),
+					])->vanisheduids->ids;
+				},
+				array_chunk($request->getUids(), self::UID_CHUNK_SIZE)
+			)
+		);
+		return $vanishedUids;
+	}
 }