diff options
author | Camila <hello@camila.codes> | 2022-02-25 19:30:15 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-02-25 19:30:15 +0300 |
commit | a892dbf50cedf0cf65446cd556f4e2b5b76f77e3 (patch) | |
tree | 133b64fb608334831718dfc9613f69ee5eb5d81b | |
parent | fc058ef9670baf7b1a6d2dfc8847d9af9d4f454d (diff) | |
parent | b8cd630072e72ddb6397b7f628e805c9392b73a1 (diff) |
Merge pull request #4312 from nextcloud/backport/4268/stable-3.4
[stable-3.4] Do not remove a folder that has files that were not uploaded yet during propagation
-rw-r--r-- | src/libsync/owncloudpropagator.cpp | 68 | ||||
-rw-r--r-- | src/libsync/owncloudpropagator.h | 2 | ||||
-rw-r--r-- | test/testsyncengine.cpp | 47 |
3 files changed, 117 insertions, 0 deletions
diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 3f13e42c4..342245701 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -413,6 +413,65 @@ void OwncloudPropagator::resetDelayedUploadTasks() _delayedTasks.clear(); } +void OwncloudPropagator::adjustDeletedFoldersWithNewChildren(SyncFileItemVector &items) +{ + /* + process each item that is new and is a directory and make sure every parent in its tree has the instruction CSYNC_INSTRUCTION_NEW + instead of CSYNC_INSTRUCTION_REMOVE + NOTE: We are iterating backwords to take advantage of optimization later, when searching for the parent of current it + */ + for (auto it = std::crbegin(items); it != std::crend(items); ++it) { + if ((*it)->_instruction != CSYNC_INSTRUCTION_NEW || (*it)->_direction != SyncFileItem::Up || !(*it)->isDirectory() || (*it)->_file == QStringLiteral("/")) { + continue; + } + + // #1 get root folder name for the current item that we need to reupload + const auto folderPathSplit = (*it)->_file.split(QLatin1Char('/'), Qt::SkipEmptyParts); + if (folderPathSplit.isEmpty()) { + continue; + } + const auto itemRootFolderName = folderPathSplit.first(); + if (itemRootFolderName.isEmpty()) { + continue; + } + // #2 iterate backwords (for optimization) and find the root folder by name + const auto itemRootFolderReverseIt = std::find_if(it, std::crend(items), [&itemRootFolderName](const auto ¤tItem) { + return currentItem->_file == itemRootFolderName; + }); + + if (itemRootFolderReverseIt == std::rend(items)) { + continue; + } + + // #3 convert reverse iterator to normal iterator + const auto itemFolderIt = (itemRootFolderReverseIt + 1).base(); + + // #4 if the root folder is set to be removed, then we will need to fix this by reuploading every folder in + // the tree, including the root + if (itemFolderIt == std::end(items)) { + continue; + } + + auto nextFolderInTreeIt = itemFolderIt; + do { + // #5 Iterate forward from the CSYNC_INSTRUCTION_NEW folder's root, and make sure every folder in it's tree is set to CSYNC_INSTRUCTION_NEW + if ((*nextFolderInTreeIt)->isDirectory() + && (*nextFolderInTreeIt)->_instruction == CSYNC_INSTRUCTION_REMOVE + && (*nextFolderInTreeIt)->_direction == SyncFileItem::Down + && (*it)->_file.startsWith(QString((*nextFolderInTreeIt)->_file) + QLatin1Char('/'))) { + + qCWarning(lcPropagator) << "WARNING: New directory to upload " << (*it)->_file + << "is in the removed directories tree " << (*nextFolderInTreeIt)->_file + << " This should not happen! But, we are going to reupload the entire folder structure."; + + (*nextFolderInTreeIt)->_instruction = CSYNC_INSTRUCTION_NEW; + (*nextFolderInTreeIt)->_direction = SyncFileItem::Up; + } + ++nextFolderInTreeIt; + } while (nextFolderInTreeIt != std::end(items) && (*nextFolderInTreeIt)->_file != (*it)->_file); + } +} + qint64 OwncloudPropagator::smallFileSize() { const qint64 smallFileSize = 100 * 1024; //default to 1 MB. Not dynamic right now. @@ -448,6 +507,15 @@ void OwncloudPropagator::start(SyncFileItemVector &&items) items.end()); } + QStringList files; + + for (const auto &item : items) { + files.push_back(item->_file); + } + + // process each item that is new and is a directory and make sure every parent in its tree has the instruction NEW instead of REMOVE + adjustDeletedFoldersWithNewChildren(items); + resetDelayedUploadTasks(); _rootJob.reset(new PropagateRootDirectory(this)); QStack<QPair<QString /* directory name */, PropagateDirectory * /* job */>> directories; diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h index 28f3994b0..79d017e82 100644 --- a/src/libsync/owncloudpropagator.h +++ b/src/libsync/owncloudpropagator.h @@ -665,6 +665,8 @@ private: void resetDelayedUploadTasks(); + static void adjustDeletedFoldersWithNewChildren(SyncFileItemVector &items); + AccountPtr _account; QScopedPointer<PropagateRootDirectory> _rootJob; SyncOptions _syncOptions; diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp index 462fc84f4..e407b6682 100644 --- a/test/testsyncengine.cpp +++ b/test/testsyncengine.cpp @@ -151,6 +151,53 @@ private slots: QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); } + void testLocalDeleteWithReuploadForNewLocalFiles() + { + FakeFolder fakeFolder{FileInfo{}}; + + // create folders hierarchy with some nested dirs and files + fakeFolder.localModifier().mkdir("A"); + fakeFolder.localModifier().insert("A/existingfile_A.txt", 100); + fakeFolder.localModifier().mkdir("A/B"); + fakeFolder.localModifier().insert("A/B/existingfile_B.data", 100); + fakeFolder.localModifier().mkdir("A/B/C"); + fakeFolder.localModifier().mkdir("A/B/C/c1"); + fakeFolder.localModifier().mkdir("A/B/C/c1/c2"); + fakeFolder.localModifier().insert("A/B/C/c1/c2/existingfile_C2.md", 100); + + QVERIFY(fakeFolder.syncOnce()); + + // make sure everything is uploaded + QVERIFY(fakeFolder.currentRemoteState().find("A/B/C/c1/c2")); + QVERIFY(fakeFolder.currentRemoteState().find("A/existingfile_A.txt")); + QVERIFY(fakeFolder.currentRemoteState().find("A/B/existingfile_B.data")); + QVERIFY(fakeFolder.currentRemoteState().find("A/B/C/c1/c2/existingfile_C2.md")); + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + + // remove a folder "A" on the server + fakeFolder.remoteModifier().remove("A"); + + // put new files and folders into a local folder "A" + fakeFolder.localModifier().insert("A/B/C/c1/c2/newfile.txt", 100); + fakeFolder.localModifier().insert("A/B/C/c1/c2/Readme.data", 100); + fakeFolder.localModifier().mkdir("A/B/C/c1/c2/newfiles"); + fakeFolder.localModifier().insert("A/B/C/c1/c2/newfiles/newfile.txt", 100); + fakeFolder.localModifier().insert("A/B/C/c1/c2/newfiles/Readme.data", 100); + + QVERIFY(fakeFolder.syncOnce()); + // make sure new files and folders are uploaded (restored) + QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/Readme.data")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/newfiles/newfile.txt")); + QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/newfiles/Readme.data")); + // and the old files are removed + QVERIFY(!fakeFolder.currentLocalState().find("A/existingfile_A.txt")); + QVERIFY(!fakeFolder.currentLocalState().find("A/B/existingfile_B.data")); + QVERIFY(!fakeFolder.currentLocalState().find("A/B/C/c1/c2/existingfile_C2.md")); + + QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); + } + void testEmlLocalChecksum() { FakeFolder fakeFolder{FileInfo{}}; fakeFolder.localModifier().insert("a1.eml", 64, 'A'); |