Merge pull request #32708 from nextcloud/unencrypted-size-24

[24] store unencrypted size in the unencrypted_size column

10 files changed, 164 insertions, 56 deletions

diff --git a/lib/private/Files/Cache/Cache.php b/lib/private/Files/Cache/Cache.php
index 949079dfa22..dc6ab6dae22 100644
--- a/lib/private/Files/Cache/Cache.php
+++ b/lib/private/Files/Cache/Cache.php
@@ -37,6 +37,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
+
 namespace OC\Files\Cache;
 
 use Doctrine\DBAL\Exception\UniqueConstraintViolationException;
@@ -188,6 +189,7 @@ class Cache implements ICache {
 		$data['fileid'] = (int)$data['fileid'];
 		$data['parent'] = (int)$data['parent'];
 		$data['size'] = 0 + $data['size'];
+		$data['unencrypted_size'] = 0 + ($data['unencrypted_size'] ?? 0);
 		$data['mtime'] = (int)$data['mtime'];
 		$data['storage_mtime'] = (int)$data['storage_mtime'];
 		$data['encryptedVersion'] = (int)$data['encrypted'];
@@ -428,7 +430,7 @@ class Cache implements ICache {
 	protected function normalizeData(array $data): array {
 		$fields = [
 			'path', 'parent', 'name', 'mimetype', 'size', 'mtime', 'storage_mtime', 'encrypted',
-			'etag', 'permissions', 'checksum', 'storage'];
+			'etag', 'permissions', 'checksum', 'storage', 'unencrypted_size'];
 		$extensionFields = ['metadata_etag', 'creation_time', 'upload_time'];
 
 		$doNotCopyStorageMTime = false;
@@ -873,18 +875,32 @@ class Cache implements ICache {
 			$id = $entry['fileid'];
 
 			$query = $this->getQueryBuilder();
-			$query->selectAlias($query->func()->sum('size'), 'f1')
-				->selectAlias($query->func()->min('size'), 'f2')
+			$query->select('size', 'unencrypted_size')
 				->from('filecache')
-				->whereStorageId($this->getNumericStorageId())
 				->whereParent($id);
 
 			$result = $query->execute();
-			$row = $result->fetch();
+			$rows = $result->fetchAll();
 			$result->closeCursor();
 
-			if ($row) {
-				[$sum, $min] = array_values($row);
+			if ($rows) {
+				$sizes = array_map(function (array $row) {
+					return (int)$row['size'];
+				}, $rows);
+				$unencryptedOnlySizes = array_map(function (array $row) {
+					return (int)$row['unencrypted_size'];
+				}, $rows);
+				$unencryptedSizes = array_map(function (array $row) {
+					return (int)(($row['unencrypted_size'] > 0) ? $row['unencrypted_size']: $row['size']);
+				}, $rows);
+
+				$sum = array_sum($sizes);
+				$min = min($sizes);
+
+				$unencryptedSum = array_sum($unencryptedSizes);
+				$unencryptedMin = min($unencryptedSizes);
+				$unencryptedMax = max($unencryptedOnlySizes);
+
 				$sum = 0 + $sum;
 				$min = 0 + $min;
 				if ($min === -1) {
@@ -892,8 +908,23 @@ class Cache implements ICache {
 				} else {
 					$totalSize = $sum;
 				}
+				if ($unencryptedMin === -1 || $min === -1) {
+					$unencryptedTotal = $unencryptedMin;
+				} else {
+					$unencryptedTotal = $unencryptedSum;
+				}
 				if ($entry['size'] !== $totalSize) {
-					$this->update($id, ['size' => $totalSize]);
+					// only set unencrypted size for a folder if any child entries have it set
+					if ($unencryptedMax > 0) {
+						$this->update($id, [
+							'size' => $totalSize,
+							'unencrypted_size' => $unencryptedTotal,
+						]);
+					} else {
+						$this->update($id, [
+							'size' => $totalSize,
+						]);
+					}
 				}
 			}
 		}
diff --git a/lib/private/Files/Cache/CacheEntry.php b/lib/private/Files/Cache/CacheEntry.php
index 12f0273fb6e..8ac76acf6d1 100644
--- a/lib/private/Files/Cache/CacheEntry.php
+++ b/lib/private/Files/Cache/CacheEntry.php
@@ -132,4 +132,12 @@ class CacheEntry implements ICacheEntry {
 	public function __clone() {
 		$this->data = array_merge([], $this->data);
 	}
+
+	public function getUnencryptedSize(): int {
+		if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+			return $this->data['unencrypted_size'];
+		} else {
+			return $this->data['size'];
+		}
+	}
 }
diff --git a/lib/private/Files/Cache/CacheQueryBuilder.php b/lib/private/Files/Cache/CacheQueryBuilder.php
index b5a9101877c..fbcc9a05557 100644
--- a/lib/private/Files/Cache/CacheQueryBuilder.php
+++ b/lib/private/Files/Cache/CacheQueryBuilder.php
@@ -44,7 +44,7 @@ class CacheQueryBuilder extends QueryBuilder {
 	public function selectFileCache(string $alias = null) {
 		$name = $alias ? $alias : 'filecache';
 		$this->select("$name.fileid", 'storage', 'path', 'path_hash', "$name.parent", 'name', 'mimetype', 'mimepart', 'size', 'mtime',
-			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time')
+			'storage_mtime', 'encrypted', 'etag', 'permissions', 'checksum', 'metadata_etag', 'creation_time', 'upload_time', 'unencrypted_size')
 			->from('filecache', $name)
 			->leftJoin($name, 'filecache_extended', 'fe', $this->expr()->eq("$name.fileid", 'fe.fileid'));
 
diff --git a/lib/private/Files/Cache/Propagator.php b/lib/private/Files/Cache/Propagator.php
index 270b2b013f5..4086701f011 100644
--- a/lib/private/Files/Cache/Propagator.php
+++ b/lib/private/Files/Cache/Propagator.php
@@ -24,6 +24,8 @@
 
 namespace OC\Files\Cache;
 
+use OC\DB\QueryBuilder\QueryFunction;
+use OC\Files\Storage\Wrapper\Encryption;
 use OCP\DB\QueryBuilder\IQueryBuilder;
 use OCP\Files\Cache\IPropagator;
 use OCP\Files\Storage\IReliableEtagStorage;
@@ -113,6 +115,20 @@ class Propagator implements IPropagator {
 				->andWhere($builder->expr()->in('path_hash', $hashParams))
 				->andWhere($builder->expr()->gt('size', $builder->expr()->literal(-1, IQueryBuilder::PARAM_INT)));
 
+			if ($this->storage->instanceOfStorage(Encryption::class)) {
+				// in case of encryption being enabled after some files are already uploaded, some entries will have an unencrypted_size of 0 and a non-zero size
+				$eq = $builder->expr()->eq('unencrypted_size', $builder->expr()->literal(0, IQueryBuilder::PARAM_INT));
+				$sizeColumn = $builder->getColumnName('size');
+				$unencryptedSizeColumn = $builder->getColumnName('unencrypted_size');
+				$builder->set('unencrypted_size', $builder->func()->greatest(
+					$builder->func()->add(
+						new QueryFunction("CASE WHEN $eq THEN $unencryptedSizeColumn ELSE $sizeColumn END"),
+						$builder->createNamedParameter($sizeDifference)
+					),
+					$builder->createNamedParameter(-1, IQueryBuilder::PARAM_INT)
+				));
+			}
+
 			$builder->execute();
 		}
 	}
diff --git a/lib/private/Files/FileInfo.php b/lib/private/Files/FileInfo.php
index 6389544184f..5912eefcf9c 100644
--- a/lib/private/Files/FileInfo.php
+++ b/lib/private/Files/FileInfo.php
@@ -101,7 +101,11 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 		$this->data = $data;
 		$this->mount = $mount;
 		$this->owner = $owner;
-		$this->rawSize = $this->data['size'] ?? 0;
+		if (isset($this->data['unencrypted_size'])) {
+			$this->rawSize = $this->data['unencrypted_size'];
+		} else {
+			$this->rawSize = $this->data['size'] ?? 0;
+		}
 	}
 
 	public function offsetSet($offset, $value): void {
@@ -208,7 +212,12 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	public function getSize($includeMounts = true) {
 		if ($includeMounts) {
 			$this->updateEntryfromSubMounts();
-			return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+
+			if (isset($this->data['unencrypted_size']) && $this->data['unencrypted_size'] > 0) {
+				return $this->data['unencrypted_size'];
+			} else {
+				return isset($this->data['size']) ? 0 + $this->data['size'] : 0;
+			}
 		} else {
 			return $this->rawSize;
 		}
@@ -386,7 +395,19 @@ class FileInfo implements \OCP\Files\FileInfo, \ArrayAccess {
 	 * @param string $entryPath full path of the child entry
 	 */
 	public function addSubEntry($data, $entryPath) {
-		$this->data['size'] += isset($data['size']) ? $data['size'] : 0;
+		if (!$data) {
+			return;
+		}
+		$hasUnencryptedSize = isset($data['unencrypted_size']) && $data['unencrypted_size'] > 0;
+		if ($hasUnencryptedSize) {
+			$subSize = $data['unencrypted_size'];
+		} else {
+			$subSize = $data['size'] ?: 0;
+		}
+		$this->data['size'] += $subSize;
+		if ($hasUnencryptedSize) {
+			$this->data['unencrypted_size'] += $subSize;
+		}
 		if (isset($data['mtime'])) {
 			$this->data['mtime'] = max($this->data['mtime'], $data['mtime']);
 		}
diff --git a/lib/private/Files/Storage/Wrapper/Encryption.php b/lib/private/Files/Storage/Wrapper/Encryption.php
index 4cfe932cc9f..d5bf929101f 100644
--- a/lib/private/Files/Storage/Wrapper/Encryption.php
+++ b/lib/private/Files/Storage/Wrapper/Encryption.php
@@ -33,6 +33,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>
  *
  */
+
 namespace OC\Files\Storage\Wrapper;
 
 use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
@@ -41,6 +42,7 @@ use OC\Encryption\Util;
 use OC\Files\Cache\CacheEntry;
 use OC\Files\Filesystem;
 use OC\Files\Mount\Manager;
+use OC\Files\ObjectStore\ObjectStoreStorage;
 use OC\Files\Storage\LocalTempFileTrait;
 use OC\Memcache\ArrayCache;
 use OCP\Encryption\Exceptions\GenericEncryptionException;
@@ -139,28 +141,36 @@ class Encryption extends Wrapper {
 			$size = $this->unencryptedSize[$fullPath];
 			// update file cache
 			if ($info instanceof ICacheEntry) {
-				$info = $info->getData();
 				$info['encrypted'] = $info['encryptedVersion'];
 			} else {
 				if (!is_array($info)) {
 					$info = [];
 				}
 				$info['encrypted'] = true;
+				$info = new CacheEntry($info);
 			}
 
-			$info['size'] = $size;
-			$this->getCache()->put($path, $info);
+			if ($size !== $info->getUnencryptedSize()) {
+				$this->getCache()->update($info->getId(), [
+					'unencrypted_size' => $size
+				]);
+			}
 
 			return $size;
 		}
 
 		if (isset($info['fileid']) && $info['encrypted']) {
-			return $this->verifyUnencryptedSize($path, $info['size']);
+			return $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
 		}
 
 		return $this->storage->filesize($path);
 	}
 
+	/**
+	 * @param string $path
+	 * @param array $data
+	 * @return array
+	 */
 	private function modifyMetaData(string $path, array $data): array {
 		$fullPath = $this->getFullPath($path);
 		$info = $this->getCache()->get($path);
@@ -170,7 +180,7 @@ class Encryption extends Wrapper {
 			$data['size'] = $this->unencryptedSize[$fullPath];
 		} else {
 			if (isset($info['fileid']) && $info['encrypted']) {
-				$data['size'] = $this->verifyUnencryptedSize($path, $info['size']);
+				$data['size'] = $this->verifyUnencryptedSize($path, $info->getUnencryptedSize());
 				$data['encrypted'] = true;
 			}
 		}
@@ -478,7 +488,7 @@ class Encryption extends Wrapper {
 	 *
 	 * @return int unencrypted size
 	 */
-	protected function verifyUnencryptedSize($path, $unencryptedSize) {
+	protected function verifyUnencryptedSize(string $path, int $unencryptedSize): int {
 		$size = $this->storage->filesize($path);
 		$result = $unencryptedSize;
 
@@ -510,7 +520,7 @@ class Encryption extends Wrapper {
 	 *
 	 * @return int calculated unencrypted size
 	 */
-	protected function fixUnencryptedSize($path, $size, $unencryptedSize) {
+	protected function fixUnencryptedSize(string $path, int $size, int $unencryptedSize): int {
 		$headerSize = $this->getHeaderSize($path);
 		$header = $this->getHeader($path);
 		$encryptionModule = $this->getEncryptionModule($path);
@@ -581,7 +591,9 @@ class Encryption extends Wrapper {
 		$cache = $this->storage->getCache();
 		if ($cache) {
 			$entry = $cache->get($path);
-			$cache->update($entry['fileid'], ['size' => $newUnencryptedSize]);
+			$cache->update($entry['fileid'], [
+				'unencrypted_size' => $newUnencryptedSize
+			]);
 		}
 
 		return $newUnencryptedSize;
@@ -621,7 +633,12 @@ class Encryption extends Wrapper {
 	 * @param bool $preserveMtime
 	 * @return bool
 	 */
-	public function moveFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = true) {
+	public function moveFromStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime = true
+	) {
 		if ($sourceStorage === $this) {
 			return $this->rename($sourceInternalPath, $targetInternalPath);
 		}
@@ -656,7 +673,13 @@ class Encryption extends Wrapper {
 	 * @param bool $isRename
 	 * @return bool
 	 */
-	public function copyFromStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime = false, $isRename = false) {
+	public function copyFromStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime = false,
+		$isRename = false
+	) {
 
 		// TODO clean this up once the underlying moveFromStorage in OC\Files\Storage\Wrapper\Common is fixed:
 		// - call $this->storage->copyFromStorage() instead of $this->copyBetweenStorage
@@ -676,7 +699,13 @@ class Encryption extends Wrapper {
 	 * @param bool $isRename
 	 * @param bool $keepEncryptionVersion
 	 */
-	private function updateEncryptedVersion(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, $keepEncryptionVersion) {
+	private function updateEncryptedVersion(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$isRename,
+		$keepEncryptionVersion
+	) {
 		$isEncrypted = $this->encryptionManager->isEnabled() && $this->shouldEncrypt($targetInternalPath);
 		$cacheInformation = [
 			'encrypted' => $isEncrypted,
@@ -725,7 +754,13 @@ class Encryption extends Wrapper {
 	 * @return bool
 	 * @throws \Exception
 	 */
-	private function copyBetweenStorage(Storage\IStorage $sourceStorage, $sourceInternalPath, $targetInternalPath, $preserveMtime, $isRename) {
+	private function copyBetweenStorage(
+		Storage\IStorage $sourceStorage,
+		$sourceInternalPath,
+		$targetInternalPath,
+		$preserveMtime,
+		$isRename
+	) {
 
 		// for versions we have nothing to do, because versions should always use the
 		// key from the original file. Just create a 1:1 copy and done
@@ -743,7 +778,7 @@ class Encryption extends Wrapper {
 				if (isset($info['encrypted']) && $info['encrypted'] === true) {
 					$this->updateUnencryptedSize(
 						$this->getFullPath($targetInternalPath),
-						$info['size']
+						$info->getUnencryptedSize()
 					);
 				}
 				$this->updateEncryptedVersion($sourceStorage, $sourceInternalPath, $targetInternalPath, $isRename, true);
@@ -808,13 +843,6 @@ class Encryption extends Wrapper {
 		return (bool)$result;
 	}
 
-	/**
-	 * get the path to a local version of the file.
-	 * The local version of the file can be temporary and doesn't have to be persistent across requests
-	 *
-	 * @param string $path
-	 * @return string
-	 */
 	public function getLocalFile($path) {
 		if ($this->encryptionManager->isEnabled()) {
 			$cachedFile = $this->getCachedFile($path);
@@ -825,11 +853,6 @@ class Encryption extends Wrapper {
 		return $this->storage->getLocalFile($path);
 	}
 
-	/**
-	 * Returns the wrapped storage's value for isLocal()
-	 *
-	 * @return bool wrapped storage's isLocal() value
-	 */
 	public function isLocal() {
 		if ($this->encryptionManager->isEnabled()) {
 			return false;
@@ -837,15 +860,11 @@ class Encryption extends Wrapper {
 		return $this->storage->isLocal();
 	}
 
-	/**
-	 * see https://www.php.net/manual/en/function.stat.php
-	 * only the following keys are required in the result: size and mtime
-	 *
-	 * @param string $path
-	 * @return array
-	 */
 	public function stat($path) {
 		$stat = $this->storage->stat($path);
+		if (!$stat) {
+			return false;
+		}
 		$fileSize = $this->filesize($path);
 		$stat['size'] = $fileSize;
 		$stat[7] = $fileSize;
@@ -853,14 +872,6 @@ class Encryption extends Wrapper {
 		return $stat;
 	}
 
-	/**
-	 * see https://www.php.net/manual/en/function.hash.php
-	 *
-	 * @param string $type
-	 * @param string $path
-	 * @param bool $raw
-	 * @return string
-	 */
 	public function hash($type, $path, $raw = false) {
 		$fh = $this->fopen($path, 'rb');
 		$ctx = hash_init($type);
@@ -1068,6 +1079,13 @@ class Encryption extends Wrapper {
 		[$count, $result] = \OC_Helper::streamCopy($stream, $target);
 		fclose($stream);
 		fclose($target);
+
+		// object store, stores the size after write and doesn't update this during scan
+		// manually store the unencrypted size
+		if ($result && $this->getWrapperStorage()->instanceOfStorage(ObjectStoreStorage::class)) {
+			$this->getCache()->put($path, ['unencrypted_size' => $count]);
+		}
+
 		return $count;
 	}
 }
diff --git a/lib/public/Files/Cache/ICacheEntry.php b/lib/public/Files/Cache/ICacheEntry.php
index 17eecf89ddb..e1e8129394c 100644
--- a/lib/public/Files/Cache/ICacheEntry.php
+++ b/lib/public/Files/Cache/ICacheEntry.php
@@ -162,4 +162,14 @@ interface ICacheEntry extends ArrayAccess {
 	 * @since 18.0.0
 	 */
 	public function getUploadTime(): ?int;
+
+	/**
+	 * Get the unencrypted size
+	 *
+	 * This might be different from the result of getSize
+	 *
+	 * @return int
+	 * @since 25.0.0
+	 */
+	public function getUnencryptedSize(): int;
 }
diff --git a/tests/lib/Files/ObjectStore/S3Test.php b/tests/lib/Files/ObjectStore/S3Test.php
index a7a95d53375..fd451dc3c01 100644
--- a/tests/lib/Files/ObjectStore/S3Test.php
+++ b/tests/lib/Files/ObjectStore/S3Test.php
@@ -176,7 +176,7 @@ class S3Test extends ObjectStoreTest {
 
 		// end of file reached
 		fseek($result, $size);
-		self:self::assertTrue(feof($result));
+		self::assertTrue(feof($result));
 
 		$this->assertNoUpload('testfilesizes');
 	}
diff --git a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php
index d26e5c499e7..ebb97a25c77 100644
--- a/tests/lib/Files/Storage/Wrapper/EncryptionTest.php
+++ b/tests/lib/Files/Storage/Wrapper/EncryptionTest.php
@@ -5,6 +5,7 @@ namespace Test\Files\Storage\Wrapper;
 use OC\Encryption\Exceptions\ModuleDoesNotExistsException;
 use OC\Encryption\Update;
 use OC\Encryption\Util;
+use OC\Files\Cache\CacheEntry;
 use OC\Files\Storage\Temporary;
 use OC\Files\Storage\Wrapper\Encryption;
 use OC\Files\View;
@@ -259,7 +260,7 @@ class EncryptionTest extends Storage {
 			->method('get')
 			->willReturnCallback(
 				function ($path) use ($encrypted) {
-					return ['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1];
+					return new CacheEntry(['encrypted' => $encrypted, 'path' => $path, 'size' => 0, 'fileid' => 1]);
 				}
 			);
 
@@ -332,7 +333,7 @@ class EncryptionTest extends Storage {
 			->disableOriginalConstructor()->getMock();
 		$cache->expects($this->any())
 			->method('get')
-			->willReturn(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]);
+			->willReturn(new CacheEntry(['encrypted' => true, 'path' => '/test.txt', 'size' => 0, 'fileid' => 1]));
 
 		$this->instance = $this->getMockBuilder('\OC\Files\Storage\Wrapper\Encryption')
 			->setConstructorArgs(
@@ -910,7 +911,7 @@ class EncryptionTest extends Storage {
 		if ($copyResult) {
 			$cache->expects($this->once())->method('get')
 				->with($sourceInternalPath)
-				->willReturn(['encrypted' => $encrypted, 'size' => 42]);
+				->willReturn(new CacheEntry(['encrypted' => $encrypted, 'size' => 42]));
 			if ($encrypted) {
 				$instance->expects($this->once())->method('updateUnencryptedSize')
 					->with($mountPoint . $targetInternalPath, 42);
diff --git a/tests/lib/HelperStorageTest.php b/tests/lib/HelperStorageTest.php
index 6d7ea513d3f..d3f480502b2 100644
--- a/tests/lib/HelperStorageTest.php
+++ b/tests/lib/HelperStorageTest.php
@@ -104,6 +104,9 @@ class HelperStorageTest extends \Test\TestCase {
 		$extStorage->file_put_contents('extfile.txt', 'abcdefghijklmnopq');
 		$extStorage->getScanner()->scan(''); // update root size
 
+		$config = \OC::$server->getConfig();
+		$config->setSystemValue('quota_include_external_storage', false);
+
 		\OC\Files\Filesystem::mount($extStorage, [], '/' . $this->user . '/files/ext');
 
 		$storageInfo = \OC_Helper::getStorageInfo('');