Upload: asynchronious operations

Implements https://github.com/owncloud/core/pull/31851

3 files changed, 304 insertions, 22 deletions

diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 3c192bd6b..89c2b2712 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -50,6 +50,7 @@ owncloud_add_test(SyncFileStatusTracker "syncenginetestutils.h")
 owncloud_add_test(Download "syncenginetestutils.h")
 owncloud_add_test(ChunkingNg "syncenginetestutils.h")
 owncloud_add_test(Zsync "syncenginetestutils.h")
+owncloud_add_test(AsyncOp "syncenginetestutils.h")
 owncloud_add_test(UploadReset "syncenginetestutils.h")
 owncloud_add_test(AllFilesDeleted "syncenginetestutils.h")
 owncloud_add_test(Blacklist "syncenginetestutils.h")
diff --git a/test/syncenginetestutils.h b/test/syncenginetestutils.h
index 00f75134e..a15c01bab 100644
--- a/test/syncenginetestutils.h
+++ b/test/syncenginetestutils.h
@@ -45,7 +45,7 @@ inline QString getFilePathFromUrl(const QUrl &url) {
 
 
 inline QString generateEtag() {
-    return QString::number(QDateTime::currentDateTimeUtc().toMSecsSinceEpoch(), 16);
+    return QString::number(QDateTime::currentDateTimeUtc().toMSecsSinceEpoch(), 16) + QByteArray::number(qrand(), 16);
 }
 inline QByteArray generateFileId() {
     return QByteArray::number(qrand(), 16);
@@ -266,7 +266,7 @@ public:
             auto file = it->find(std::move(pathComponents).subComponents(), invalidateEtags);
             if (file && invalidateEtags) {
                 // Update parents on the way back
-                etag = file->etag;
+                etag = generateEtag();
             }
             return file;
         }
@@ -336,7 +336,6 @@ public:
     QMap<QString, FileInfo> children;
     QString parentPath;
 
-private:
     FileInfo *findInvalidatingEtags(PathComponents pathComponents) {
         return find(std::move(pathComponents), true);
     }
@@ -459,24 +458,25 @@ public:
         setUrl(request.url());
         setOperation(op);
         open(QIODevice::ReadOnly);
+        fileInfo = perform(remoteRootFileInfo, request, putPayload);
+        QMetaObject::invokeMethod(this, "respond", Qt::QueuedConnection);
+    }
 
+    static FileInfo *perform(FileInfo &remoteRootFileInfo, const QNetworkRequest &request, const QByteArray &putPayload)
+    {
         QString fileName = getFilePathFromUrl(request.url());
         Q_ASSERT(!fileName.isEmpty());
-        if ((fileInfo = remoteRootFileInfo.find(fileName))) {
+        FileInfo *fileInfo = remoteRootFileInfo.find(fileName);
+        if (fileInfo) {
             fileInfo->size = putPayload.size();
             fileInfo->contentChar = putPayload.at(0);
         } else {
             // Assume that the file is filled with the same character
             fileInfo = remoteRootFileInfo.create(fileName, putPayload.size(), putPayload.at(0));
         }
-
-        if (!fileInfo) {
-            abort();
-            return;
-        }
         fileInfo->lastModified = OCC::Utility::qDateTimeFromTime_t(request.rawHeader("X-OC-Mtime").toLongLong());
         remoteRootFileInfo.find(fileName, /*invalidate_etags=*/true);
-        QMetaObject::invokeMethod(this, "respond", Qt::QueuedConnection);
+        return fileInfo;
     }
 
     Q_INVOKABLE virtual void respond()
@@ -744,7 +744,16 @@ public:
         setUrl(request.url());
         setOperation(op);
         open(QIODevice::ReadOnly);
+        fileInfo = perform(uploadsFileInfo, remoteRootFileInfo, request);
+        if (!fileInfo) {
+            QTimer::singleShot(0, this, &FakeChunkMoveReply::respondPreconditionFailed);
+        } else {
+            QTimer::singleShot(0, this, &FakeChunkMoveReply::respond);
+        }
+    }
 
+    static FileInfo *perform(FileInfo &uploadsFileInfo, FileInfo &remoteRootFileInfo, const QNetworkRequest &request)
+    {
         QString source = getFilePathFromUrl(request.url());
         bool zsync = false;
         Q_ASSERT(!source.isEmpty());
@@ -783,7 +792,7 @@ public:
             ++count;
             prev = chunkName.toLongLong() + x.size;
         }
-        QCOMPARE(sourceFolder->children.count(), count); // There should not be holes or extra files
+        Q_ASSERT(sourceFolder->children.count() == count); // There should not be holes or extra files
 
         // For zsync, get the size from the header, and allow no-chunk uploads (shrinking files)
         if (zsync) {
@@ -795,13 +804,12 @@ public:
         }
 
         // NOTE: This does not actually assemble the file data from the chunks!
-
-        if ((fileInfo = remoteRootFileInfo.find(fileName))) {
-            QVERIFY(request.hasRawHeader("If")); // The client should put this header
+        FileInfo *fileInfo = remoteRootFileInfo.find(fileName);
+        if (fileInfo) {
+            Q_ASSERT(request.hasRawHeader("If")); // The client should put this header
             if (request.rawHeader("If") != QByteArray("<" + request.rawHeader("Destination") +
                                                 "> ([\"" + fileInfo->etag.toLatin1() + "\"])")) {
-                QMetaObject::invokeMethod(this, "respondPreconditionFailed", Qt::QueuedConnection);
-                return;
+                return nullptr;
             }
             fileInfo->size = size;
             fileInfo->contentChar = payload;
@@ -810,15 +818,10 @@ public:
             // Assume that the file is filled with the same character
             fileInfo = remoteRootFileInfo.create(fileName, size, payload);
         }
-
-        if (!fileInfo) {
-            abort();
-            return;
-        }
         fileInfo->lastModified = OCC::Utility::qDateTimeFromTime_t(request.rawHeader("X-OC-Mtime").toLongLong());
         remoteRootFileInfo.find(fileName, /*invalidate_etags=*/true);
 
-        QTimer::singleShot(0, this, &FakeChunkMoveReply::respond);
+        return fileInfo;
     }
 
     Q_INVOKABLE virtual void respond()
@@ -940,6 +943,48 @@ public:
     qint64 readData(char *, qint64) override { return 0; }
 };
 
+class FakePayloadReply : public QNetworkReply
+{
+    Q_OBJECT
+public:
+    FakePayloadReply(QNetworkAccessManager::Operation op, const QNetworkRequest &request,
+        const QByteArray &body, QObject *parent)
+        : QNetworkReply{ parent }
+        , _body(body)
+    {
+        setRequest(request);
+        setUrl(request.url());
+        setOperation(op);
+        open(QIODevice::ReadOnly);
+        QTimer::singleShot(10, this, &FakePayloadReply::respond);
+    }
+
+    void respond()
+    {
+        setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 200);
+        setHeader(QNetworkRequest::ContentLengthHeader, _body.size());
+        emit metaDataChanged();
+        emit readyRead();
+        setFinished(true);
+        emit finished();
+    }
+
+    void abort() override {}
+    qint64 readData(char *buf, qint64 max) override
+    {
+        max = qMin<qint64>(max, _body.size());
+        memcpy(buf, _body.constData(), max);
+        _body = _body.mid(max);
+        return max;
+    }
+    qint64 bytesAvailable() const override
+    {
+        return _body.size();
+    }
+    QByteArray _body;
+};
+
+
 class FakeErrorReply : public QNetworkReply
 {
     Q_OBJECT
diff --git a/test/testasyncop.cpp b/test/testasyncop.cpp
new file mode 100644
index 000000000..930fd382f
--- /dev/null
+++ b/test/testasyncop.cpp
@@ -0,0 +1,236 @@
+/*
+ *    This software is in the public domain, furnished "as is", without technical
+ *    support, and with no warranty, express or implied, as to its usefulness for
+ *    any purpose.
+ *
+ */
+
+#include <QtTest>
+#include "syncenginetestutils.h"
+#include <syncengine.h>
+
+using namespace OCC;
+
+class FakeAsyncReply : public QNetworkReply
+{
+    Q_OBJECT
+    QByteArray _pollLocation;
+
+public:
+    FakeAsyncReply(const QByteArray &pollLocation, QNetworkAccessManager::Operation op, const QNetworkRequest &request, QObject *parent)
+        : QNetworkReply{ parent }
+        , _pollLocation(pollLocation)
+    {
+        setRequest(request);
+        setUrl(request.url());
+        setOperation(op);
+        open(QIODevice::ReadOnly);
+
+        QMetaObject::invokeMethod(this, "respond", Qt::QueuedConnection);
+    }
+
+    Q_INVOKABLE void respond()
+    {
+        setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 202);
+        setRawHeader("OC-JobStatus-Location", _pollLocation);
+        emit metaDataChanged();
+        emit finished();
+    }
+
+    void abort() override {}
+    qint64 readData(char *, qint64) override { return 0; }
+};
+
+
+class TestAsyncOp : public QObject
+{
+    Q_OBJECT
+
+private slots:
+
+    void asyncUploadOperations()
+    {
+        FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() };
+        fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" } } } });
+        // Reduce max chunk size a bit so we get more chunks
+        SyncOptions options;
+        options._maxChunkSize = 20 * 1000;
+        fakeFolder.syncEngine().setSyncOptions(options);
+        int nGET = 0;
+
+        // This test is made of several testcases.
+        // the testCases maps a filename to a couple of callback.
+        // When a file is uploaded, the fake server will always return the 202 code, and will set
+        // the `perform` functor to what needs to be done to complete the transaction.
+        // The testcase consist of the `pollRequest` which will be called when the sync engine
+        // calls the poll url.
+        struct TestCase
+        {
+            using PollRequest_t = std::function<QNetworkReply *(TestCase *, const QNetworkRequest &request)>;
+            PollRequest_t pollRequest;
+            std::function<FileInfo *()> perform = nullptr;
+        };
+        QHash<QString, TestCase> testCases;
+
+        fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *outgoingData) -> QNetworkReply * {
+            auto path = request.url().path();
+
+            if (op == QNetworkAccessManager::GetOperation && path.startsWith("/async-poll/")) {
+                auto file = path.mid(sizeof("/async-poll/") - 1);
+                Q_ASSERT(testCases.contains(file));
+                auto &testCase = testCases[file];
+                return testCase.pollRequest(&testCase, request);
+            }
+
+            if (op == QNetworkAccessManager::PutOperation && !path.contains("/uploads/")) {
+                // Not chunking
+                auto file = getFilePathFromUrl(request.url());
+                Q_ASSERT(testCases.contains(file));
+                auto &testCase = testCases[file];
+                Q_ASSERT(!testCase.perform);
+                auto putPayload = outgoingData->readAll();
+                testCase.perform = [putPayload, request, &fakeFolder] {
+                    return FakePutReply::perform(fakeFolder.remoteModifier(), request, putPayload);
+                };
+                return new FakeAsyncReply("/async-poll/" + file.toUtf8(), op, request, &fakeFolder.syncEngine());
+            } else if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
+                QString file = getFilePathFromUrl(QUrl::fromEncoded(request.rawHeader("Destination")));
+                Q_ASSERT(testCases.contains(file));
+                auto &testCase = testCases[file];
+                Q_ASSERT(!testCase.perform);
+                testCase.perform = [request, &fakeFolder] {
+                    return FakeChunkMoveReply::perform(fakeFolder.uploadState(), fakeFolder.remoteModifier(), request);
+                };
+                return new FakeAsyncReply("/async-poll/" + file.toUtf8(), op, request, &fakeFolder.syncEngine());
+            } else if (op == QNetworkAccessManager::GetOperation) {
+                nGET++;
+            }
+            return nullptr;
+        });
+
+
+        // Callback to be used to finalize the transaction and return the success
+        auto successCallback = [](TestCase *tc, const QNetworkRequest &request) {
+            tc->pollRequest = [](auto...) -> QNetworkReply * { std::abort(); }; // shall no longer be called
+            FileInfo *info = tc->perform();
+            QByteArray body = "{ \"status\":\"finished\", \"ETag\":\"\\\"" + info->etag.toUtf8() + "\\\"\", \"fileId\":\"" + info->fileId + "\"}\n";
+            return new FakePayloadReply(QNetworkAccessManager::GetOperation, request, body, nullptr);
+        };
+        // Callback that never finishes
+        auto waitForeverCallback = [](TestCase *, const QNetworkRequest &request) {
+            QByteArray body = "{\"status\":\"started\"}\n";
+            return new FakePayloadReply(QNetworkAccessManager::GetOperation, request, body, nullptr);
+        };
+        // Callback that simulate an error.
+        auto errorCallback = [](TestCase *tc, const QNetworkRequest &request) {
+            tc->pollRequest = [](auto...) -> QNetworkReply * { std::abort(); }; // shall no longer be called;
+            QByteArray body = "{\"status\":\"error\",\"errorCode\":500,\"errorMessage\":\"TestingErrors\"}\n";
+            return new FakePayloadReply(QNetworkAccessManager::GetOperation, request, body, nullptr);
+        };
+        // This lambda takes another functor as a parameter, and returns a callback that will
+        // tell the client needs to poll again, and further call to the poll url will call the
+        // given callback
+        auto waitAndChain = [](const TestCase::PollRequest_t &chain) {
+            return [chain](TestCase *tc, const QNetworkRequest &request) {
+                tc->pollRequest = chain;
+                QByteArray body = "{\"status\":\"started\"}\n";
+                return new FakePayloadReply(QNetworkAccessManager::GetOperation, request, body, nullptr);
+            };
+        };
+
+        // Create a testcase by creating a file of a given size locally and assigning it a callback
+        auto insertFile = [&](const QString &file, int size, TestCase::PollRequest_t cb) {
+            fakeFolder.localModifier().insert(file, size);
+            testCases[file] = { std::move(cb) };
+        };
+        fakeFolder.localModifier().mkdir("success");
+        insertFile("success/chunked_success", options._maxChunkSize * 3, successCallback);
+        insertFile("success/single_success", 300, successCallback);
+        insertFile("success/chunked_patience", options._maxChunkSize * 3,
+            waitAndChain(waitAndChain(successCallback)));
+        insertFile("success/single_patience", 300,
+            waitAndChain(waitAndChain(successCallback)));
+        fakeFolder.localModifier().mkdir("err");
+        insertFile("err/chunked_error", options._maxChunkSize * 3, errorCallback);
+        insertFile("err/single_error", 300, errorCallback);
+        insertFile("err/chunked_error2", options._maxChunkSize * 3, waitAndChain(errorCallback));
+        insertFile("err/single_error2", 300, waitAndChain(errorCallback));
+
+        // First sync should finish by itself.
+        // All the things in "success/" should be transfered, the things in "err/" not
+        QVERIFY(!fakeFolder.syncOnce());
+        QCOMPARE(nGET, 0);
+        QCOMPARE(*fakeFolder.currentLocalState().find("success"),
+            *fakeFolder.currentRemoteState().find("success"));
+        testCases.clear();
+        testCases["err/chunked_error"] = { successCallback };
+        testCases["err/chunked_error2"] = { successCallback };
+        testCases["err/single_error"] = { successCallback };
+        testCases["err/single_error2"] = { successCallback };
+
+        fakeFolder.localModifier().mkdir("waiting");
+        insertFile("waiting/small", 300, waitForeverCallback);
+        insertFile("waiting/willNotConflict", 300, waitForeverCallback);
+        insertFile("waiting/big", options._maxChunkSize * 3,
+            waitAndChain(waitAndChain([&](TestCase *tc, const QNetworkRequest &request) {
+                QTimer::singleShot(0, &fakeFolder.syncEngine(), &SyncEngine::abort);
+                return waitAndChain(waitForeverCallback)(tc, request);
+            })));
+
+        fakeFolder.syncJournal().wipeErrorBlacklist();
+
+        // This second sync will redo the files that had errors
+        // But the waiting folder will not complete before it is aborted.
+        QVERIFY(!fakeFolder.syncOnce());
+        QCOMPARE(nGET, 0);
+        QCOMPARE(*fakeFolder.currentLocalState().find("err"),
+            *fakeFolder.currentRemoteState().find("err"));
+
+        testCases["waiting/small"].pollRequest = waitAndChain(waitAndChain(successCallback));
+        testCases["waiting/big"].pollRequest = waitAndChain(successCallback);
+        testCases["waiting/willNotConflict"].pollRequest =
+            [&fakeFolder, &successCallback](TestCase *tc, const QNetworkRequest &request) {
+                auto &remoteModifier = fakeFolder.remoteModifier(); // successCallback destroys the capture
+                auto reply = successCallback(tc, request);
+                // This is going to succeed, and after we just change the file.
+                // This should not be a conflict, but this should be downloaded in the
+                // next sync
+                remoteModifier.appendByte("waiting/willNotConflict");
+                return reply;
+            };
+
+
+        int nPUT = 0;
+        int nMOVE = 0;
+        int nDELETE = 0;
+        fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * {
+            auto path = request.url().path();
+            if (op == QNetworkAccessManager::GetOperation && path.startsWith("/async-poll/")) {
+                auto file = path.mid(sizeof("/async-poll/") - 1);
+                Q_ASSERT(testCases.contains(file));
+                auto &testCase = testCases[file];
+                return testCase.pollRequest(&testCase, request);
+            } else if (op == QNetworkAccessManager::PutOperation) {
+                nPUT++;
+            } else if (op == QNetworkAccessManager::GetOperation) {
+                nGET++;
+            } else if (op == QNetworkAccessManager::DeleteOperation) {
+                nDELETE++;
+            } else if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") {
+                nMOVE++;
+            }
+            return nullptr;
+        });
+
+        // This last sync will do the waiting stuff
+        QVERIFY(fakeFolder.syncOnce());
+        QCOMPARE(nGET, 1); // "waiting/willNotConflict"
+        QCOMPARE(nPUT, 0);
+        QCOMPARE(nMOVE, 0);
+        QCOMPARE(nDELETE, 0);
+        QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState());
+    }
+};
+
+QTEST_GUILESS_MAIN(TestAsyncOp)
+#include "testasyncop.moc"