Merge pull request #4312 from nextcloud/backport/4268/stable-3.4

[stable-3.4] Do not remove a folder that has files that were not uploaded yet during propagation

3 files changed, 117 insertions, 0 deletions

diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp
index 3f13e42c4..342245701 100644
--- a/src/libsync/owncloudpropagator.cpp
+++ b/src/libsync/owncloudpropagator.cpp
@@ -413,6 +413,65 @@ void OwncloudPropagator::resetDelayedUploadTasks()
     _delayedTasks.clear();
 }
 
+void OwncloudPropagator::adjustDeletedFoldersWithNewChildren(SyncFileItemVector &items)
+{
+    /* 
+       process each item that is new and is a directory and make sure every parent in its tree has the instruction CSYNC_INSTRUCTION_NEW
+       instead of CSYNC_INSTRUCTION_REMOVE
+       NOTE: We are iterating backwords to take advantage of optimization later, when searching for the parent of current it
+    */
+    for (auto it = std::crbegin(items); it != std::crend(items); ++it) {
+        if ((*it)->_instruction != CSYNC_INSTRUCTION_NEW || (*it)->_direction != SyncFileItem::Up || !(*it)->isDirectory() || (*it)->_file == QStringLiteral("/")) {
+            continue;
+        }
+
+        // #1 get root folder name for the current item that we need to reupload
+        const auto folderPathSplit = (*it)->_file.split(QLatin1Char('/'), Qt::SkipEmptyParts);
+        if (folderPathSplit.isEmpty()) {
+            continue;
+        }
+        const auto itemRootFolderName = folderPathSplit.first();
+        if (itemRootFolderName.isEmpty()) {
+            continue;
+        }
+        // #2 iterate backwords (for optimization) and find the root folder by name
+        const auto itemRootFolderReverseIt = std::find_if(it, std::crend(items), [&itemRootFolderName](const auto &currentItem) {
+            return currentItem->_file == itemRootFolderName;
+        });
+
+        if (itemRootFolderReverseIt == std::rend(items)) {
+            continue;
+        }
+
+        // #3 convert reverse iterator to normal iterator
+        const auto itemFolderIt = (itemRootFolderReverseIt + 1).base();
+
+        // #4 if the root folder is set to be removed, then we will need to fix this by reuploading every folder in
+        // the tree, including the root
+        if (itemFolderIt == std::end(items)) {
+            continue;
+        }
+
+        auto nextFolderInTreeIt = itemFolderIt;
+        do {
+            // #5 Iterate forward from the CSYNC_INSTRUCTION_NEW folder's root, and make sure every folder in it's tree is set to CSYNC_INSTRUCTION_NEW
+            if ((*nextFolderInTreeIt)->isDirectory()
+                && (*nextFolderInTreeIt)->_instruction == CSYNC_INSTRUCTION_REMOVE
+                && (*nextFolderInTreeIt)->_direction == SyncFileItem::Down
+                && (*it)->_file.startsWith(QString((*nextFolderInTreeIt)->_file) + QLatin1Char('/'))) {
+
+                qCWarning(lcPropagator) << "WARNING: New directory to upload " << (*it)->_file
+                    << "is in the removed directories tree " << (*nextFolderInTreeIt)->_file
+                    << " This should not happen! But, we are going to reupload the entire folder structure.";
+
+                (*nextFolderInTreeIt)->_instruction = CSYNC_INSTRUCTION_NEW;
+                (*nextFolderInTreeIt)->_direction = SyncFileItem::Up;
+            }
+            ++nextFolderInTreeIt;
+        } while (nextFolderInTreeIt != std::end(items) && (*nextFolderInTreeIt)->_file != (*it)->_file);
+    }
+}
+
 qint64 OwncloudPropagator::smallFileSize()
 {
     const qint64 smallFileSize = 100 * 1024; //default to 1 MB. Not dynamic right now.
@@ -448,6 +507,15 @@ void OwncloudPropagator::start(SyncFileItemVector &&items)
             items.end());
     }
 
+    QStringList files;
+
+    for (const auto &item : items) {
+        files.push_back(item->_file);
+    }
+
+    // process each item that is new and is a directory and make sure every parent in its tree has the instruction NEW instead of REMOVE
+    adjustDeletedFoldersWithNewChildren(items);
+
     resetDelayedUploadTasks();
     _rootJob.reset(new PropagateRootDirectory(this));
     QStack<QPair<QString /* directory name */, PropagateDirectory * /* job */>> directories;
diff --git a/src/libsync/owncloudpropagator.h b/src/libsync/owncloudpropagator.h
index 28f3994b0..79d017e82 100644
--- a/src/libsync/owncloudpropagator.h
+++ b/src/libsync/owncloudpropagator.h
@@ -665,6 +665,8 @@ private:
 
     void resetDelayedUploadTasks();
 
+    static void adjustDeletedFoldersWithNewChildren(SyncFileItemVector &items);
+
     AccountPtr _account;
     QScopedPointer<PropagateRootDirectory> _rootJob;
     SyncOptions _syncOptions;
diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp
index 462fc84f4..e407b6682 100644
--- a/test/testsyncengine.cpp
+++ b/test/testsyncengine.cpp
@@ -151,6 +151,53 @@ private slots:
         QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
     }
 
+    void testLocalDeleteWithReuploadForNewLocalFiles()
+    {
+        FakeFolder fakeFolder{FileInfo{}};
+
+        // create folders hierarchy with some nested dirs and files
+        fakeFolder.localModifier().mkdir("A");
+        fakeFolder.localModifier().insert("A/existingfile_A.txt", 100);
+        fakeFolder.localModifier().mkdir("A/B");
+        fakeFolder.localModifier().insert("A/B/existingfile_B.data", 100);
+        fakeFolder.localModifier().mkdir("A/B/C");
+        fakeFolder.localModifier().mkdir("A/B/C/c1");
+        fakeFolder.localModifier().mkdir("A/B/C/c1/c2");
+        fakeFolder.localModifier().insert("A/B/C/c1/c2/existingfile_C2.md", 100);
+
+        QVERIFY(fakeFolder.syncOnce());
+
+        // make sure everything is uploaded
+        QVERIFY(fakeFolder.currentRemoteState().find("A/B/C/c1/c2"));
+        QVERIFY(fakeFolder.currentRemoteState().find("A/existingfile_A.txt"));
+        QVERIFY(fakeFolder.currentRemoteState().find("A/B/existingfile_B.data"));
+        QVERIFY(fakeFolder.currentRemoteState().find("A/B/C/c1/c2/existingfile_C2.md"));
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+
+        // remove a folder "A" on the server
+        fakeFolder.remoteModifier().remove("A");
+
+        // put new files and folders into a local folder "A"
+        fakeFolder.localModifier().insert("A/B/C/c1/c2/newfile.txt", 100);
+        fakeFolder.localModifier().insert("A/B/C/c1/c2/Readme.data", 100);
+        fakeFolder.localModifier().mkdir("A/B/C/c1/c2/newfiles");
+        fakeFolder.localModifier().insert("A/B/C/c1/c2/newfiles/newfile.txt", 100);
+        fakeFolder.localModifier().insert("A/B/C/c1/c2/newfiles/Readme.data", 100);
+
+        QVERIFY(fakeFolder.syncOnce());
+        // make sure new files and folders are uploaded (restored)
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/Readme.data"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/newfiles/newfile.txt"));
+        QVERIFY(fakeFolder.currentLocalState().find("A/B/C/c1/c2/newfiles/Readme.data"));
+        // and the old files are removed
+        QVERIFY(!fakeFolder.currentLocalState().find("A/existingfile_A.txt"));
+        QVERIFY(!fakeFolder.currentLocalState().find("A/B/existingfile_B.data"));
+        QVERIFY(!fakeFolder.currentLocalState().find("A/B/C/c1/c2/existingfile_C2.md"));
+
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+    }
+
     void testEmlLocalChecksum() {
         FakeFolder fakeFolder{FileInfo{}};
         fakeFolder.localModifier().insert("a1.eml", 64, 'A');