diff options
author | Vincent Petry <vincent@nextcloud.com> | 2022-08-17 10:21:30 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-08-17 10:21:30 +0300 |
commit | 082432e01cb3cb614139d285b769cb3f68310748 (patch) | |
tree | fa2c98c6fb2e96c4208dde42223059f90a0ecefc | |
parent | 58fb55febe662f2e208ae71f596b1a8881ed2298 (diff) | |
parent | 457822c1444d3e8b543e029e729b35727a66fad0 (diff) |
Merge pull request #33551 from nextcloud/scanner-dont-update-same-size
don't update the folder size if we know it hasn't changed
-rw-r--r-- | lib/private/Files/Cache/Scanner.php | 13 | ||||
-rw-r--r-- | lib/private/Files/ObjectStore/NoopScanner.php | 2 | ||||
-rw-r--r-- | tests/lib/Files/Cache/ScannerTest.php | 78 |
3 files changed, 47 insertions, 46 deletions
diff --git a/lib/private/Files/Cache/Scanner.php b/lib/private/Files/Cache/Scanner.php index fb32b64c012..4799c3bff7d 100644 --- a/lib/private/Files/Cache/Scanner.php +++ b/lib/private/Files/Cache/Scanner.php @@ -339,7 +339,7 @@ class Scanner extends BasicEmitter implements IScanner { try { $data = $this->scanFile($path, $reuse, -1, null, $lock); if ($data and $data['mimetype'] === 'httpd/unix-directory') { - $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock); + $size = $this->scanChildren($path, $recursive, $reuse, $data['fileid'], $lock, $data); $data['size'] = $size; } } finally { @@ -376,9 +376,10 @@ class Scanner extends BasicEmitter implements IScanner { * @param int $reuse * @param int $folderId id for the folder to be scanned * @param bool $lock set to false to disable getting an additional read lock during scanning + * @param array $data the data of the folder before (re)scanning the children * @return int the size of the scanned folder or -1 if the size is unknown at this stage */ - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { + protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) { if ($reuse === -1) { $reuse = ($recursive === self::SCAN_SHALLOW) ? self::REUSE_ETAG | self::REUSE_SIZE : self::REUSE_ETAG; } @@ -397,7 +398,8 @@ class Scanner extends BasicEmitter implements IScanner { $size += $childSize; } } - if ($this->cacheActive) { + $oldSize = $data['size'] ?? null; + if ($this->cacheActive && $oldSize !== $size) { $this->cache->update($folderId, ['size' => $size]); } $this->emit('\OC\Files\Cache\Scanner', 'postScanFolder', [$path, $this->storageId]); @@ -409,6 +411,11 @@ class Scanner extends BasicEmitter implements IScanner { $existingChildren = $this->getExistingChildren($folderId); $newChildren = iterator_to_array($this->storage->getDirectoryContent($path)); + if (count($existingChildren) === 0 && count($newChildren) === 0) { + // no need to do a transaction + return []; + } + if ($this->useTransactions) { \OC::$server->getDatabaseConnection()->beginTransaction(); } diff --git a/lib/private/Files/ObjectStore/NoopScanner.php b/lib/private/Files/ObjectStore/NoopScanner.php index 3b8cbdb18bb..bdfc93758d4 100644 --- a/lib/private/Files/ObjectStore/NoopScanner.php +++ b/lib/private/Files/ObjectStore/NoopScanner.php @@ -68,7 +68,7 @@ class NoopScanner extends Scanner { * @param array $folderData existing cache data for the folder to be scanned * @return int the size of the scanned folder or -1 if the size is unknown at this stage */ - protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true) { + protected function scanChildren($path, $recursive = self::SCAN_RECURSIVE, $reuse = -1, $folderId = null, $lock = true, array $data = []) { return 0; } diff --git a/tests/lib/Files/Cache/ScannerTest.php b/tests/lib/Files/Cache/ScannerTest.php index 29e619cd09b..e4c052f6025 100644 --- a/tests/lib/Files/Cache/ScannerTest.php +++ b/tests/lib/Files/Cache/ScannerTest.php @@ -8,7 +8,14 @@ namespace Test\Files\Cache; +use OC; +use OC\Files\Cache\Cache; use OC\Files\Cache\CacheEntry; +use OC\Files\Cache\Scanner; +use OC\Files\Storage\Storage; +use OC\Files\Storage\Temporary; +use OCP\Files\Cache\IScanner; +use Test\TestCase; /** * Class ScannerTest @@ -17,34 +24,21 @@ use OC\Files\Cache\CacheEntry; * * @package Test\Files\Cache */ -class ScannerTest extends \Test\TestCase { - /** - * @var \OC\Files\Storage\Storage $storage - */ - private $storage; - - /** - * @var \OC\Files\Cache\Scanner $scanner - */ - private $scanner; - - /** - * @var \OC\Files\Cache\Cache $cache - */ - private $cache; +class ScannerTest extends TestCase { + private Storage $storage; + private Scanner $scanner; + private Cache $cache; protected function setUp(): void { parent::setUp(); - $this->storage = new \OC\Files\Storage\Temporary([]); - $this->scanner = new \OC\Files\Cache\Scanner($this->storage); - $this->cache = new \OC\Files\Cache\Cache($this->storage); + $this->storage = new Temporary([]); + $this->scanner = new Scanner($this->storage); + $this->cache = new Cache($this->storage); } protected function tearDown(): void { - if ($this->cache) { - $this->cache->clear(); - } + $this->cache->clear(); parent::tearDown(); } @@ -60,7 +54,7 @@ class ScannerTest extends \Test\TestCase { $this->assertEquals($cachedData['mimetype'], 'text/plain'); $this->assertNotEquals($cachedData['parent'], -1); //parent folders should be scanned automatically - $data = file_get_contents(\OC::$SERVERROOT . '/core/img/logo/logo.png'); + $data = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png'); $this->storage->file_put_contents('foo.png', $data); $this->scanner->scanFile('foo.png'); @@ -74,7 +68,7 @@ class ScannerTest extends \Test\TestCase { $data = "dummy file data\n"; $this->storage->file_put_contents('foo🙈.txt', $data); - if (\OC::$server->getDatabaseConnection()->supports4ByteText()) { + if (OC::$server->getDatabaseConnection()->supports4ByteText()) { $this->assertNotNull($this->scanner->scanFile('foo🙈.txt')); $this->assertTrue($this->cache->inCache('foo🙈.txt'), true); @@ -98,7 +92,7 @@ class ScannerTest extends \Test\TestCase { private function fillTestFolders() { $textData = "dummy file data\n"; - $imgData = file_get_contents(\OC::$SERVERROOT . '/core/img/logo/logo.png'); + $imgData = file_get_contents(OC::$SERVERROOT . '/core/img/logo/logo.png'); $this->storage->mkdir('folder'); $this->storage->file_put_contents('foo.txt', $textData); $this->storage->file_put_contents('foo.png', $imgData); @@ -130,7 +124,7 @@ class ScannerTest extends \Test\TestCase { public function testShallow() { $this->fillTestFolders(); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('', IScanner::SCAN_SHALLOW); $this->assertEquals($this->cache->inCache(''), true); $this->assertEquals($this->cache->inCache('foo.txt'), true); $this->assertEquals($this->cache->inCache('foo.png'), true); @@ -143,7 +137,7 @@ class ScannerTest extends \Test\TestCase { $this->assertEquals(-1, $cachedDataFolder['size']); $this->assertEquals(-1, $cachedDataFolder2['size']); - $this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('folder', IScanner::SCAN_SHALLOW); $cachedDataFolder2 = $this->cache->get('folder'); @@ -160,7 +154,7 @@ class ScannerTest extends \Test\TestCase { $this->storage->mkdir('folder2'); $this->storage->file_put_contents('folder2/bar.txt', 'foobar'); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('', IScanner::SCAN_SHALLOW); $this->assertFalse($this->cache->inCache('folder/bar.txt')); $this->assertFalse($this->cache->inCache('folder/2bar.txt')); $cachedData = $this->cache->get(''); @@ -182,7 +176,7 @@ class ScannerTest extends \Test\TestCase { $this->storage->mkdir('folder2'); $this->storage->file_put_contents('folder2/bar.txt', 'foobar'); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('', IScanner::SCAN_SHALLOW); $this->assertFalse($this->cache->inCache('folder/bar.txt')); $this->assertFalse($this->cache->inCache('folder/2bar.txt')); $this->assertFalse($this->cache->inCache('folder2/bar.txt')); @@ -191,7 +185,7 @@ class ScannerTest extends \Test\TestCase { $cachedData = $this->cache->get(''); $this->assertEquals(-1, $cachedData['size']); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE_INCOMPLETE, \OC\Files\Cache\Scanner::REUSE_ETAG | \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->scanner->scan('', IScanner::SCAN_RECURSIVE_INCOMPLETE, IScanner::REUSE_ETAG | IScanner::REUSE_SIZE); $this->assertTrue($this->cache->inCache('folder/bar.txt')); $this->assertTrue($this->cache->inCache('folder/bar.txt')); @@ -248,7 +242,7 @@ class ScannerTest extends \Test\TestCase { $oldData = $this->cache->get(''); $this->storage->unlink('folder/bar.txt'); $this->cache->put('folder', ['mtime' => $this->storage->filemtime('folder'), 'storage_mtime' => $this->storage->filemtime('folder')]); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_SIZE); $newData = $this->cache->get(''); $this->assertIsString($oldData['etag']); $this->assertIsString($newData['etag']); @@ -256,25 +250,25 @@ class ScannerTest extends \Test\TestCase { $this->assertEquals($oldData['size'], $newData['size']); $oldData = $newData; - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG); + $this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); $this->assertEquals(-1, $newData['size']); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE); + $this->scanner->scan('', IScanner::SCAN_RECURSIVE); $oldData = $this->cache->get(''); $this->assertNotEquals(-1, $oldData['size']); - $this->scanner->scanFile('', \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->scanner->scanFile('', IScanner::REUSE_ETAG + IScanner::REUSE_SIZE); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); $this->assertEquals($oldData['size'], $newData['size']); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_RECURSIVE, \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->scanner->scan('', IScanner::SCAN_RECURSIVE, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); $this->assertEquals($oldData['size'], $newData['size']); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG + \OC\Files\Cache\Scanner::REUSE_SIZE); + $this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG + IScanner::REUSE_SIZE); $newData = $this->cache->get(''); $this->assertSame($oldData['etag'], $newData['etag']); $this->assertEquals($oldData['size'], $newData['size']); @@ -286,7 +280,7 @@ class ScannerTest extends \Test\TestCase { $this->scanner->scan(''); $this->assertTrue($this->cache->inCache('foo.txt')); $this->storage->unlink('foo.txt'); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('', IScanner::SCAN_SHALLOW); $this->assertFalse($this->cache->inCache('foo.txt')); } @@ -296,7 +290,7 @@ class ScannerTest extends \Test\TestCase { $this->scanner->scan(''); $this->assertTrue($this->cache->inCache('folder/bar.txt')); $this->storage->rmdir('/folder'); - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('', IScanner::SCAN_SHALLOW); $this->assertFalse($this->cache->inCache('folder')); $this->assertFalse($this->cache->inCache('folder/bar.txt')); } @@ -317,7 +311,7 @@ class ScannerTest extends \Test\TestCase { $this->scanner->scan('folder/bar.txt'); // manipulate etag to simulate an empty etag - $this->scanner->scan('', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG); + $this->scanner->scan('', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG); /** @var CacheEntry $data0 */ $data0 = $this->cache->get('folder/bar.txt'); $this->assertIsString($data0['etag']); @@ -329,7 +323,7 @@ class ScannerTest extends \Test\TestCase { $this->cache->put('folder/bar.txt', $data0->getData()); // rescan - $this->scanner->scan('folder/bar.txt', \OC\Files\Cache\Scanner::SCAN_SHALLOW, \OC\Files\Cache\Scanner::REUSE_ETAG); + $this->scanner->scan('folder/bar.txt', IScanner::SCAN_SHALLOW, IScanner::REUSE_ETAG); // verify cache content $newData0 = $this->cache->get('folder/bar.txt'); @@ -344,7 +338,7 @@ class ScannerTest extends \Test\TestCase { $oldFolderId = $this->cache->getId('folder'); // delete the folder without removing the children - $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query = OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId))); $query->execute(); @@ -370,7 +364,7 @@ class ScannerTest extends \Test\TestCase { $oldFolderId = $this->cache->getId('folder'); // delete the folder without removing the children - $query = \OC::$server->getDatabaseConnection()->getQueryBuilder(); + $query = OC::$server->getDatabaseConnection()->getQueryBuilder(); $query->delete('filecache') ->where($query->expr()->eq('fileid', $query->createNamedParameter($oldFolderId))); $query->execute(); @@ -379,7 +373,7 @@ class ScannerTest extends \Test\TestCase { $this->assertEquals($oldFolderId, $cachedData['parent']); $this->assertFalse($this->cache->inCache('folder')); - $this->scanner->scan('folder', \OC\Files\Cache\Scanner::SCAN_SHALLOW); + $this->scanner->scan('folder', IScanner::SCAN_SHALLOW); $this->assertTrue($this->cache->inCache('folder')); $newFolderId = $this->cache->getId('folder'); |