lock during cron jobs

Signed-off-by: Maxence Lange <maxence@artificial-owl.com>

10 files changed, 187 insertions, 20 deletions

diff --git a/lib/Cron/Backup.php b/lib/Cron/Backup.php
index 552a8fe..adb023a 100644
--- a/lib/Cron/Backup.php
+++ b/lib/Cron/Backup.php
@@ -33,6 +33,7 @@ namespace OCA\Backup\Cron;
 
 use ArtificialOwl\MySmallPhpTools\Traits\Nextcloud\nc23\TNC23Logger;
 use OC\BackgroundJob\TimedJob;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Service\ConfigService;
 use OCA\Backup\Service\CronService;
 use OCA\Backup\Service\PointService;
@@ -87,26 +88,36 @@ class Backup extends TimedJob {
 	 * @param $argument
 	 */
 	protected function run($argument): void {
-		if (!$this->cronService->isRealCron()) {
+		if (!$this->cronService->isRunnable()) {
 			return;
 		}
 
+		try {
+			$this->cronService->lockCron(false);
+			$this->manage();
+			$this->cronService->unlockCron();
+		} catch (JobsTimeSlotException $e) {
+			return;
+		}
+	}
+
+
+	/**
+	 *
+	 */
+	private function manage(): void {
 		$time = time();
 		if ($this->configService->getAppValueInt(ConfigService::MOCKUP_DATE) > 0) {
 			$time = $this->configService->getAppValueInt(ConfigService::MOCKUP_DATE);
 			$this->configService->setAppValueInt(ConfigService::MOCKUP_DATE, 0);
 		}
 
-		if (!$this->cronService->verifyTime($time)) {
-			return;
-		}
-
 		$this->runBackup($time);
 	}
 
 
 	/**
-	 *
+	 * @param int $time
 	 */
 	private function runBackup(int $time): void {
 		if ($this->cronService->verifyFullBackup($time)) {
diff --git a/lib/Cron/Event.php b/lib/Cron/Event.php
index 1a98cd2..e9d35f5 100644
--- a/lib/Cron/Event.php
+++ b/lib/Cron/Event.php
@@ -95,7 +95,7 @@ class Event extends TimedJob {
 	 * @param $argument
 	 */
 	protected function run($argument) {
-		if (!$this->cronService->isRealCron()) {
+		if (!$this->cronService->isRunnable()) {
 			return;
 		}
 
diff --git a/lib/Cron/Manage.php b/lib/Cron/Manage.php
index 60ad420..d8027ef 100644
--- a/lib/Cron/Manage.php
+++ b/lib/Cron/Manage.php
@@ -34,6 +34,7 @@ namespace OCA\Backup\Cron;
 use OC\BackgroundJob\TimedJob;
 use OCA\Backup\Exceptions\ExternalFolderNotFoundException;
 use OCA\Backup\Model\RestoringPoint;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Service\ConfigService;
 use OCA\Backup\Service\CronService;
 use OCA\Backup\Service\ExternalFolderService;
@@ -109,16 +110,32 @@ class Manage extends TimedJob {
 	 * @param $argument
 	 */
 	protected function run($argument) {
-		if (!$this->cronService->isRealCron()) {
+		if (!$this->cronService->isRunnable()) {
 			return;
 		}
 
+		try {
+			$this->cronService->lockCron(false);
+			$this->manage();
+			$this->cronService->unlockCron();
+		} catch (JobsTimeSlotException $e) {
+		}
+	}
+
+
+	/**
+	 * @throws JobsTimeSlotException
+	 */
+	private function manage() {
 		$generateLogs = $this->configService->getAppValueBool(ConfigService::GENERATE_LOGS);
 
 		// TODO: purge old restoring points.
 		$this->cronService->purgeRestoringPoints();
 		$this->cronService->purgeRemoteRestoringPoints();
 
+		// next steps are only available during night shift
+		$this->cronService->lockCron();
+
 		// uploading
 		foreach ($this->pointService->getLocalRestoringPoints() as $point) {
 			if ($point->isArchive()) {
@@ -130,11 +147,14 @@ class Manage extends TimedJob {
 					$this->outputService->openFile($point, 'Manage Background Job (uploading)');
 				}
 				$this->uploadService->uploadPoint($point);
+			} catch (JobsTimeSlotException $e) {
+				break;
 			} catch (Throwable $e) {
 			}
 		}
 
 		// packing
+		$this->cronService->lockCron();
 		foreach ($this->pointService->getLocalRestoringPoints() as $point) {
 			if ($point->isArchive()) {
 				continue;
@@ -151,34 +171,35 @@ class Manage extends TimedJob {
 					$this->outputService->openFile($point, 'Manage Background Job (packing)');
 				}
 				$this->packService->packPoint($point);
+			} catch (JobsTimeSlotException $e) {
+				break;
 			} catch (Throwable $e) {
 			}
 		}
 
-		// next step are only executed during the night shift
-		if (!$this->cronService->verifyTime()) {
-			return;
-		}
-
-
 		// regenerate local health
+		$this->cronService->lockCron();
 		foreach ($this->pointService->getLocalRestoringPoints() as $point) {
 			if ($point->hasHealth()
 				&& $point->getHealth()->getChecked() > time() - self::DELAY_CHECK_HEALTH) {
 				continue;
 			}
 			try {
+				$this->cronService->lockCron();
 				$this->pointService->initBaseFolder($point);
 				if ($generateLogs) {
 					$this->outputService->openFile($point, 'Manage Background Job (health)');
 				}
 
 				$this->pointService->generateHealth($point, true);
+			} catch (JobsTimeSlotException $e) {
+				break;
 			} catch (Throwable $e) {
 			}
 		}
 
 		// regenerate health on ExternalFolder
+		$this->cronService->lockCron();
 		foreach ($this->externalFolderService->getAll() as $external) {
 			try {
 				foreach ($this->externalFolderService->getRestoringPoints($external) as $point) {
@@ -187,6 +208,7 @@ class Manage extends TimedJob {
 						continue;
 					}
 					try {
+						$this->cronService->lockCron();
 						$this->pointService->initBaseFolder($point);
 						if ($generateLogs) {
 							$this->outputService->openFile(
@@ -194,6 +216,8 @@ class Manage extends TimedJob {
 							);
 						}
 						$this->externalFolderService->getCurrentHealth($external, $point);
+					} catch (JobsTimeSlotException $e) {
+						return;
 					} catch (Throwable $e) {
 					}
 				}
diff --git a/lib/Exceptions/JobsTimeSlotException.php b/lib/Exceptions/JobsTimeSlotException.php
new file mode 100644
index 0000000..dea98c0
--- /dev/null
+++ b/lib/Exceptions/JobsTimeSlotException.php
@@ -0,0 +1,42 @@
+<?php
+
+declare(strict_types=1);
+
+
+/**
+ * Nextcloud - Backup now. Restore later.
+ *
+ * This file is licensed under the Affero General Public License version 3 or
+ * later. See the COPYING file.
+ *
+ * @author Maxence Lange <maxence@artificial-owl.com>
+ * @copyright 2021, Maxence Lange <maxence@artificial-owl.com>
+ * @license GNU AGPL version 3 or any later version
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ */
+
+
+namespace OCA\Backup\Exceptions;
+
+use Exception;
+
+/**
+ * Class JobsTimeSlotException
+ *
+ * @package OCA\Backup\Exceptions
+ */
+class JobsTimeSlotException extends Exception {
+}
diff --git a/lib/Service/ChunkService.php b/lib/Service/ChunkService.php
index 5fdfa5d..d8c6ae5 100644
--- a/lib/Service/ChunkService.php
+++ b/lib/Service/ChunkService.php
@@ -41,6 +41,7 @@ use OCA\Backup\Exceptions\ArchiveFileNotFoundException;
 use OCA\Backup\Exceptions\ArchiveNotFoundException;
 use OCA\Backup\Exceptions\BackupAppCopyException;
 use OCA\Backup\Exceptions\BackupScriptNotFoundException;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Exceptions\RestoreChunkException;
 use OCA\Backup\Exceptions\RestoringChunkNotFoundException;
 use OCA\Backup\Exceptions\RestoringDataNotFoundException;
@@ -80,6 +81,9 @@ class ChunkService {
 	/** @var EncryptService */
 	private $encryptService;
 
+	/** @var CronService */
+	private $cronService;
+
 	/** @var OutputService */
 	private $outputService;
 
@@ -98,11 +102,13 @@ class ChunkService {
 	public function __construct(
 		FilesService $filesService,
 		EncryptService $encryptService,
+		CronService $cronService,
 		OutputService $outputService,
 		ConfigService $configService
 	) {
 		$this->filesService = $filesService;
 		$this->encryptService = $encryptService;
+		$this->cronService = $cronService;
 		$this->outputService = $outputService;
 		$this->configService = $configService;
 	}
@@ -114,6 +120,8 @@ class ChunkService {
 	 * @return void
 	 * @throws ArchiveCreateException
 	 * @throws ArchiveNotFoundException
+	 * @throws NotPermittedException
+	 * @throws RestoringPointNotInitiatedException
 	 */
 	public function createChunks(RestoringPoint $point): void {
 		$this->o('> creating chunks');
@@ -123,6 +131,12 @@ class ChunkService {
 				continue;
 			}
 
+			// now would be a good place to refresh tick on lock from cronjob
+			try {
+				$this->cronService->lockCron(false);
+			} catch (JobsTimeSlotException $e) {
+			}
+
 			$this->o('  * <info>' . $data->getName() . '</info>: ', false);
 			$this->filesService->initRestoringData($data);
 			if (!$data->isLocked()) {
@@ -355,6 +369,8 @@ class ChunkService {
 	 *
 	 * @throws ArchiveCreateException
 	 * @throws ArchiveNotFoundException
+	 * @throws NotPermittedException
+	 * @throws RestoringPointNotInitiatedException
 	 */
 	private function fillChunks(RestoringPoint $point, RestoringData $data) {
 		$files = $data->getFiles();
@@ -441,6 +457,12 @@ class ChunkService {
 		$chunk = new RestoringChunk($data->getName());
 		$chunkSize = $this->configService->getAppValueInt(ConfigService::CHUNK_SIZE) * 1024 * 1024;
 
+		// now would be a good place to refresh tick on lock from cronjob
+		try {
+			$this->cronService->lockCron(false);
+		} catch (JobsTimeSlotException $e) {
+		}
+
 		$zip = $this->generateZip($point, $chunk);
 		$zipSize = 0;
 		while (($filename = array_shift($files)) !== null) {
diff --git a/lib/Service/ConfigService.php b/lib/Service/ConfigService.php
index 53de97b..1c05deb 100644
--- a/lib/Service/ConfigService.php
+++ b/lib/Service/ConfigService.php
@@ -50,6 +50,7 @@ class ConfigService {
 
 	public const CRON_ENABLED = 'cron_enabled';
 	public const LOCK = 'lock';
+	public const CRON_LOCK = 'cron_lock';
 	public const REMOTE_ENABLED = 'remote_enabled';
 	public const EXTERNAL_APPDATA = 'external_appdata';
 	public const SELF_SIGNED_CERT = 'self_signed_cert';
@@ -86,6 +87,7 @@ class ConfigService {
 	public $defaults = [
 		self::CRON_ENABLED => 1,
 		self::LOCK => 0,
+		self::CRON_LOCK => 0,
 		self::REMOTE_ENABLED => 0,
 		self::EXTERNAL_APPDATA => '{}',
 		self::SELF_SIGNED_CERT => '0',
diff --git a/lib/Service/CronService.php b/lib/Service/CronService.php
index 56ae36f..fe3b1a7 100644
--- a/lib/Service/CronService.php
+++ b/lib/Service/CronService.php
@@ -35,6 +35,7 @@ use ArtificialOwl\MySmallPhpTools\Exceptions\SignatoryException;
 use ArtificialOwl\MySmallPhpTools\Exceptions\SignatureException;
 use ArtificialOwl\MySmallPhpTools\Traits\TArrayTools;
 use OCA\Backup\Exceptions\ExternalFolderNotFoundException;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Exceptions\RemoteInstanceException;
 use OCA\Backup\Exceptions\RemoteInstanceNotFoundException;
 use OCA\Backup\Exceptions\RemoteResourceNotFoundException;
@@ -54,6 +55,7 @@ class CronService {
 
 	public const MARGIN = 1800;
 	public const HOURS_FOR_NEXT = 4000;
+	public const LOCK_TIMEOUT = 3600;
 
 
 	/** @var PointService */
@@ -75,6 +77,10 @@ class CronService {
 	private $configService;
 
 
+	/** @var bool */
+	private $ranFromCron = false;
+
+
 	/**
 	 * CronService constructor.
 	 *
@@ -402,17 +408,48 @@ class CronService {
 
 
 	/**
+	 * we assume that calling this method indicate the process was initiated from BackgroundJobs
+	 *
 	 * @return bool
 	 */
-	public function isRealCron(): bool {
-		$mode = $this->configService->getCoreValue('backgroundjobs_mode', '');
+	public function isRunnable(): bool {
+		$mode = strtolower($this->configService->getCoreValue('backgroundjobs_mode', ''));
+		if ($mode !== 'cron' && $mode !== 'webcron') {
+			return false;
+		}
 
 		if (!$this->configService->getAppValueBool(ConfigService::CRON_ENABLED)) {
 			return false;
 		}
 
 		$this->configService->setAppValueBool(ConfigService::CRON_ENABLED, true);
+		$this->ranFromCron = true;
+
+		return ($this->configService->getAppValueInt(ConfigService::CRON_LOCK) < time() - self::LOCK_TIMEOUT);
+	}
+
+
+	/**
+	 * @param bool $verifyTime
+	 *
+	 * @throws JobsTimeSlotException
+	 */
+	public function lockCron(bool $verifyTime = true): void {
+		if (!$this->ranFromCron) {
+			return;
+		}
+
+		if ($verifyTime && !$this->verifyTime()) {
+			throw new JobsTimeSlotException();
+		}
 
-		return (strtolower($mode) === 'cron' || strtolower($mode) === 'webcron');
+		$this->configService->setAppValueInt(ConfigService::CRON_LOCK, time());
+	}
+
+	/**
+	 *
+	 */
+	public function unlockCron(): void {
+		$this->configService->setAppValueInt(ConfigService::CRON_LOCK, 0);
 	}
 }
diff --git a/lib/Service/ExternalFolderService.php b/lib/Service/ExternalFolderService.php
index 4a4e9a5..4037341 100644
--- a/lib/Service/ExternalFolderService.php
+++ b/lib/Service/ExternalFolderService.php
@@ -243,7 +243,6 @@ class ExternalFolderService {
 	 * @throws ExternalFolderNotFoundException
 	 * @throws GenericFileException
 	 * @throws LockedException
-	 * @throws MetadataException
 	 * @throws NotPermittedException
 	 * @throws RestoringChunkPartNotFoundException
 	 * @throws RestoringPointException
diff --git a/lib/Service/PackService.php b/lib/Service/PackService.php
index 3969bff..32234f8 100644
--- a/lib/Service/PackService.php
+++ b/lib/Service/PackService.php
@@ -39,6 +39,7 @@ use Exception;
 use OCA\Backup\Db\PointRequest;
 use OCA\Backup\Exceptions\ArchiveNotFoundException;
 use OCA\Backup\Exceptions\EncryptionKeyException;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Exceptions\PackDecryptException;
 use OCA\Backup\Exceptions\PackEncryptException;
 use OCA\Backup\Exceptions\RestoreChunkException;
@@ -89,6 +90,9 @@ class PackService {
 	/** @var EncryptService */
 	private $encryptService;
 
+	/** @var CronService */
+	private $cronService;
+
 	/** @var OutputService */
 	private $outputService;
 
@@ -104,6 +108,7 @@ class PackService {
 	 * @param RemoteStreamService $remoteStreamService
 	 * @param ChunkService $chunkService
 	 * @param EncryptService $encryptService
+	 * @param CronService $cronService
 	 * @param OutputService $outputService
 	 * @param ConfigService $configService
 	 */
@@ -113,6 +118,7 @@ class PackService {
 		RemoteStreamService $remoteStreamService,
 		ChunkService $chunkService,
 		EncryptService $encryptService,
+		CronService $cronService,
 		OutputService $outputService,
 		ConfigService $configService
 	) {
@@ -121,6 +127,7 @@ class PackService {
 		$this->remoteStreamService = $remoteStreamService;
 		$this->chunkService = $chunkService;
 		$this->encryptService = $encryptService;
+		$this->cronService = $cronService;
 		$this->outputService = $outputService;
 		$this->configService = $configService;
 	}
@@ -134,6 +141,7 @@ class PackService {
 	 * @throws NotPermittedException
 	 * @throws RestoringPointLockException
 	 * @throws RestoringPointPackException
+	 * @throws JobsTimeSlotException
 	 */
 	public function packPoint(RestoringPoint $point, bool $force = false): void {
 		$this->o('Packing Restoring Point <info>' . $point->getId() . '</info>');
@@ -167,6 +175,7 @@ class PackService {
 				}
 
 				$this->metadataService->lock($point);
+				$this->cronService->lockCron();
 
 				try {
 					$oldChunk = null;
diff --git a/lib/Service/UploadService.php b/lib/Service/UploadService.php
index dac2ed3..cfd3c2f 100644
--- a/lib/Service/UploadService.php
+++ b/lib/Service/UploadService.php
@@ -35,6 +35,7 @@ use ArtificialOwl\MySmallPhpTools\Traits\Nextcloud\nc23\TNC23Logger;
 use ArtificialOwl\MySmallPhpTools\Traits\TStringTools;
 use Exception;
 use OCA\Backup\Exceptions\ExternalFolderNotFoundException;
+use OCA\Backup\Exceptions\JobsTimeSlotException;
 use OCA\Backup\Exceptions\RemoteInstanceException;
 use OCA\Backup\Exceptions\RemoteInstanceNotFoundException;
 use OCA\Backup\Exceptions\RemoteResourceNotFoundException;
@@ -86,6 +87,9 @@ class UploadService {
 	/** @var MetadataService */
 	private $metadataService;
 
+	/** @var CronService */
+	private $cronService;
+
 	/** @var OutputService */
 	private $outputService;
 
@@ -102,6 +106,7 @@ class UploadService {
 	 * @param RemoteService $remoteService
 	 * @param ExternalFolderService $externalFolderService
 	 * @param MetadataService $metadataService
+	 * @param CronService $cronService
 	 * @param OutputService $outputService
 	 * @param ConfigService $configService
 	 */
@@ -112,6 +117,7 @@ class UploadService {
 		RemoteService $remoteService,
 		ExternalFolderService $externalFolderService,
 		MetadataService $metadataService,
+		CronService $cronService,
 		OutputService $outputService,
 		ConfigService $configService
 	) {
@@ -121,6 +127,7 @@ class UploadService {
 		$this->remoteService = $remoteService;
 		$this->externalFolderService = $externalFolderService;
 		$this->metadataService = $metadataService;
+		$this->cronService = $cronService;
 		$this->outputService = $outputService;
 		$this->configService = $configService;
 	}
@@ -158,11 +165,12 @@ class UploadService {
 	 * @param RestoringPoint $point
 	 *
 	 * @throws ExternalFolderNotFoundException
+	 * @throws JobsTimeSlotException
 	 * @throws NotFoundException
 	 * @throws NotPermittedException
 	 * @throws RemoteInstanceNotFoundException
-	 * @throws RestoringPointPackException
 	 * @throws RestoringPointLockException
+	 * @throws RestoringPointPackException
 	 */
 	public function uploadPoint(RestoringPoint $point): void {
 		$this->initUpload($point);
@@ -177,6 +185,7 @@ class UploadService {
 	 * @param string $instance
 	 *
 	 * @throws RemoteInstanceNotFoundException
+	 * @throws JobsTimeSlotException
 	 */
 	public function uploadToRemoteInstances(RestoringPoint $point, string $instance = ''): void {
 		if (!$this->configService->isRemoteEnabled()) {
@@ -217,6 +226,8 @@ class UploadService {
 				} catch (Exception $e) {
 					$this->o('<error>cannot delete out of sync package</error>');
 				}
+			} catch (JobsTimeSlotException $e) {
+				throw $e;
 			} catch (Exception $e) {
 			}
 		}
@@ -227,6 +238,8 @@ class UploadService {
 	 * @param string $instance
 	 * @param RestoringPoint $point
 	 * @param RestoringHealth $health
+	 *
+	 * @throws JobsTimeSlotException
 	 */
 	private function uploadMissingFilesToRemoteInstance(
 		string $instance,
@@ -237,7 +250,9 @@ class UploadService {
 			if ($partHealth->getStatus() === ChunkPartHealth::STATUS_OK) {
 				continue;
 			}
-			
+
+			$this->cronService->lockCron();
+
 			if ($partHealth->getChunkName() === $partHealth->getPartName()) {
 				$this->o(
 					'  * Uploading <info>' . $partHealth->getDataName() . '</info>/<info>'
@@ -282,6 +297,7 @@ class UploadService {
 	 * @param int $storageId
 	 *
 	 * @throws ExternalFolderNotFoundException
+	 * @throws JobsTimeSlotException
 	 */
 	public function uploadToExternalFolder(RestoringPoint $point, int $storageId = 0): void {
 		$this->o('- uploading ' . $point->getId() . ' to external folders');
@@ -338,6 +354,8 @@ class UploadService {
 				} catch (Exception $e) {
 					$this->o('<error>cannot delete out of sync package</error>');
 				}
+			} catch (JobsTimeSlotException $e) {
+				throw $e;
 			} catch (Exception $e) {
 				$this->o(
 					' ! issue while checking external folder: <error>' . get_class($e) .
@@ -357,6 +375,7 @@ class UploadService {
 	 * @throws NotFoundException
 	 * @throws NotPermittedException
 	 * @throws RestoringChunkPartNotFoundException
+	 * @throws JobsTimeSlotException
 	 */
 	private function uploadMissingFilesToExternalFolder(
 		ExternalFolder $external,
@@ -369,6 +388,8 @@ class UploadService {
 				continue;
 			}
 
+			$this->cronService->lockCron();
+
 			if ($partHealth->getChunkName() === $partHealth->getPartName()) {
 				$this->o(
 					'  * Uploading <info>' . $partHealth->getDataName() . '</info>/<info>'