diff options
author | Hannah von Reth <hannah.vonreth@owncloud.com> | 2020-07-14 15:19:22 +0300 |
---|---|---|
committer | Hannah von Reth <vonreth@kde.org> | 2020-07-15 15:05:57 +0300 |
commit | bcf9f3199d77fa28dc9ad76206449f1c5afaba69 (patch) | |
tree | 8aa054c6e05d0fd5aa5d7a2a80594421e18ccc7f | |
parent | cbcaeb9c1a6445f641c80fa43384f474f941c48b (diff) |
Remove dead code
-rw-r--r-- | .gitmodules | 3 | ||||
m--------- | src/3rdparty/zsync | 0 | ||||
-rw-r--r-- | src/cmd/cmd.cpp | 10 | ||||
-rw-r--r-- | src/common/remotepermissions.h | 4 | ||||
-rw-r--r-- | src/gui/folder.cpp | 3 | ||||
-rw-r--r-- | src/gui/generalsettings.cpp | 9 | ||||
-rw-r--r-- | src/gui/generalsettings.ui | 49 | ||||
-rw-r--r-- | src/libsync/CMakeLists.txt | 32 | ||||
-rw-r--r-- | src/libsync/capabilities.cpp | 5 | ||||
-rw-r--r-- | src/libsync/capabilities.h | 1 | ||||
-rw-r--r-- | src/libsync/configfile.cpp | 25 | ||||
-rw-r--r-- | src/libsync/configfile.h | 6 | ||||
-rw-r--r-- | src/libsync/discoveryphase.cpp | 11 | ||||
-rw-r--r-- | src/libsync/progressdispatcher.h | 4 | ||||
-rw-r--r-- | src/libsync/propagatecommonzsync.cpp | 239 | ||||
-rw-r--r-- | src/libsync/propagatecommonzsync.h | 106 | ||||
-rw-r--r-- | src/libsync/propagatedownload.cpp | 68 | ||||
-rw-r--r-- | src/libsync/propagatedownload.h | 114 | ||||
-rw-r--r-- | src/libsync/propagatedownloadzsync.cpp | 335 | ||||
-rw-r--r-- | src/libsync/propagateupload.h | 12 | ||||
-rw-r--r-- | src/libsync/propagateuploadng.cpp | 267 | ||||
-rw-r--r-- | src/libsync/syncoptions.h | 6 | ||||
-rw-r--r-- | test/CMakeLists.txt | 1 | ||||
-rw-r--r-- | test/syncenginetestutils.h | 122 | ||||
-rw-r--r-- | test/testzsync.cpp | 371 |
25 files changed, 51 insertions, 1752 deletions
diff --git a/.gitmodules b/.gitmodules index 0eef34ac0..1db1256f4 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ [submodule "src/3rdparty/libcrashreporter-qt"] path = src/3rdparty/libcrashreporter-qt url = git://github.com/dschmidt/libcrashreporter-qt.git -[submodule "src/3rdparty/zsync"] - path = src/3rdparty/zsync - url = https://github.com/owncloud/zsync diff --git a/src/3rdparty/zsync b/src/3rdparty/zsync deleted file mode 160000 -Subproject 302dfbf3e1dc1cfdf99d541831c8ab0372bd91e diff --git a/src/cmd/cmd.cpp b/src/cmd/cmd.cpp index 988fd8410..bed53b987 100644 --- a/src/cmd/cmd.cpp +++ b/src/cmd/cmd.cpp @@ -195,8 +195,6 @@ void help() std::cout << " --max-sync-retries [n] Retries maximum n times (default to 3)" << std::endl; std::cout << " --uplimit [n] Limit the upload speed of files to n KB/s" << std::endl; std::cout << " --downlimit [n] Limit the download speed of files to n KB/s" << std::endl; - std::cout << " --deltasync, -ds Enable delta sync (disabled by default)" << std::endl; - std::cout << " --deltasyncmin [n] Set delta sync minimum file size to n MB (10 MiB default)" << std::endl; std::cout << " -h Sync hidden files,do not ignore them" << std::endl; std::cout << " --version, -v Display version and exit" << std::endl; std::cout << " --logdebug More verbose logging" << std::endl; @@ -277,10 +275,6 @@ void parseOptions(const QStringList &app_args, CmdOptions *options) options->uplimit = it.next().toInt() * 1000; } else if (option == "--downlimit" && !it.peekNext().startsWith("-")) { options->downlimit = it.next().toInt() * 1000; - } else if (option == "-ds" || option == "--deltasync") { - options->deltasync = true; - } else if (option == "--deltasyncmin" && !it.peekNext().startsWith("-")) { - options->deltasyncminfilesize = it.next().toLongLong() * 1024 * 1024; } else if (option == "--logdebug") { Logger::instance()->setLogFile("-"); Logger::instance()->setLogDebug(true); @@ -340,8 +334,6 @@ int main(int argc, char **argv) options.restartTimes = 3; options.uplimit = 0; options.downlimit = 0; - options.deltasync = false; - options.deltasyncminfilesize = 10 * 1024 * 1024; parseOptions(app.arguments(), &options); @@ -542,8 +534,6 @@ restart_sync: SyncOptions opt; opt.fillFromEnvironmentVariables(); opt.verifyChunkSizes(); - opt._deltaSyncEnabled = options.deltasync; - opt._deltaSyncMinFileSize = options.deltasyncminfilesize; SyncEngine engine(account, options.source_dir, folder, &db); engine.setSyncOptions(opt); engine.setIgnoreHiddenFiles(options.ignoreHiddenFiles); diff --git a/src/common/remotepermissions.h b/src/common/remotepermissions.h index cee7d6f2d..d79f85559 100644 --- a/src/common/remotepermissions.h +++ b/src/common/remotepermissions.h @@ -53,11 +53,11 @@ public: IsShared = 8, // S IsMounted = 9, // M IsMountedSub = 10, // m (internal: set if the parent dir has IsMounted) - HasZSyncMetadata = 11, // z (internal: set if remote file has zsync metadata property set) + DeprecatedRemoved = 11, // z (Deprecated, don't use) // Note: when adding support for more permissions, we need to invalid the cache in the database. // (by setting forceRemoteDiscovery in SyncJournalDb::checkConnect) - PermissionsCount = HasZSyncMetadata + PermissionsCount = DeprecatedRemoved }; /// null permissions diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 39755bf02..d629f8314 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -865,9 +865,6 @@ void Folder::setSyncOptions() opt._maxChunkSize = cfgFile.maxChunkSize(); opt._targetChunkUploadDuration = cfgFile.targetChunkUploadDuration(); - opt._deltaSyncEnabled = cfgFile.deltaSyncEnabled(); - opt._deltaSyncMinFileSize = cfgFile.deltaSyncMinFileSize(); - opt.fillFromEnvironmentVariables(); opt.verifyChunkSizes(); diff --git a/src/gui/generalsettings.cpp b/src/gui/generalsettings.cpp index ba778f9b9..5f82afbbc 100644 --- a/src/gui/generalsettings.cpp +++ b/src/gui/generalsettings.cpp @@ -62,8 +62,6 @@ GeneralSettings::GeneralSettings(QWidget *parent) connect(_ui->newFolderLimitCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings); connect(_ui->newFolderLimitSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings); connect(_ui->newExternalStorage, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings); - connect(_ui->deltaSyncCheckBox, &QAbstractButton::toggled, this, &GeneralSettings::saveMiscSettings); - connect(_ui->deltaSyncSpinBox, static_cast<void (QSpinBox::*)(int)>(&QSpinBox::valueChanged), this, &GeneralSettings::saveMiscSettings); #ifndef WITH_CRASHREPORTER _ui->crashreporterCheckBox->setVisible(false); @@ -100,9 +98,6 @@ GeneralSettings::GeneralSettings(QWidget *parent) _ui->updateChannel->hide(); #endif } - if (!theme->enableExperimentalFeatures()) { - _ui->experimentalGroupBox->hide(); - } _ui->versionLabel->setText(QStringLiteral("<a href='%1'>%1</a>").arg(MIRALL_VERSION_STRING)); QObject::connect(_ui->versionLabel, &QLabel::linkActivated, this, &GeneralSettings::showAbout); @@ -131,8 +126,6 @@ void GeneralSettings::loadMiscSettings() _ui->newFolderLimitSpinBox->setValue(newFolderLimit.second); _ui->newExternalStorage->setChecked(cfgFile.confirmExternalStorage()); _ui->monoIconsCheckBox->setChecked(cfgFile.monoIcons()); - _ui->deltaSyncCheckBox->setChecked(cfgFile.deltaSyncEnabled()); - _ui->deltaSyncSpinBox->setValue(cfgFile.deltaSyncMinFileSize() / (1024 * 1024)); } void GeneralSettings::showEvent(QShowEvent *) @@ -230,8 +223,6 @@ void GeneralSettings::saveMiscSettings() cfgFile.setNewBigFolderSizeLimit(_ui->newFolderLimitCheckBox->isChecked(), _ui->newFolderLimitSpinBox->value()); cfgFile.setConfirmExternalStorage(_ui->newExternalStorage->isChecked()); - cfgFile.setDeltaSyncEnabled(_ui->deltaSyncCheckBox->isChecked()); - cfgFile.setDeltaSyncMinFileSize(_ui->deltaSyncSpinBox->value() * 1024 * 1024); } void GeneralSettings::slotToggleLaunchOnStartup(bool enable) diff --git a/src/gui/generalsettings.ui b/src/gui/generalsettings.ui index 91e7fd0b4..abf03eb40 100644 --- a/src/gui/generalsettings.ui +++ b/src/gui/generalsettings.ui @@ -164,53 +164,6 @@ </widget> </item> <item> - <widget class="QGroupBox" name="experimentalGroupBox"> - <property name="title"> - <string>Experimental</string> - </property> - <layout class="QVBoxLayout" name="verticalLayout_4"> - <item> - <layout class="QHBoxLayout" name="horizontalLayout"> - <item> - <widget class="QCheckBox" name="deltaSyncCheckBox"> - <property name="text"> - <string>Enable Delta-Synchronization for files larger than</string> - </property> - </widget> - </item> - <item> - <widget class="QSpinBox" name="deltaSyncSpinBox"> - <property name="maximum"> - <number>999999</number> - </property> - </widget> - </item> - <item> - <widget class="QLabel" name="label_2"> - <property name="text"> - <string>MB</string> - </property> - </widget> - </item> - <item> - <spacer name="horizontalSpacer_2"> - <property name="orientation"> - <enum>Qt::Horizontal</enum> - </property> - <property name="sizeHint" stdset="0"> - <size> - <width>40</width> - <height>20</height> - </size> - </property> - </spacer> - </item> - </layout> - </item> - </layout> - </widget> - </item> - <item> <widget class="QGroupBox" name="versionGroupBox"> <property name="title"> <string>Version</string> @@ -410,8 +363,6 @@ <tabstop>newExternalStorage</tabstop> <tabstop>showInExplorerNavigationPaneCheckBox</tabstop> <tabstop>crashreporterCheckBox</tabstop> - <tabstop>deltaSyncCheckBox</tabstop> - <tabstop>deltaSyncSpinBox</tabstop> <tabstop>updateChannel</tabstop> <tabstop>restartButton</tabstop> </tabstops> diff --git a/src/libsync/CMakeLists.txt b/src/libsync/CMakeLists.txt index 6935da948..5a48a14d9 100644 --- a/src/libsync/CMakeLists.txt +++ b/src/libsync/CMakeLists.txt @@ -16,9 +16,7 @@ set(libsync_SRCS owncloudtheme.cpp progressdispatcher.cpp propagatorjobs.cpp - propagatecommonzsync.cpp propagatedownload.cpp - propagatedownloadzsync.cpp propagateupload.cpp propagateuploadv1.cpp propagateuploadng.cpp @@ -45,35 +43,6 @@ else() set (libsync_SRCS ${libsync_SRCS} creds/httpcredentials.cpp) endif() -## begin zsync - -add_library(zsync STATIC ../3rdparty/zsync/c/librcksum/hash.c - ../3rdparty/zsync/c/librcksum/md4.c - ../3rdparty/zsync/c/librcksum/range.c - ../3rdparty/zsync/c/librcksum/rsum.c - ../3rdparty/zsync/c/librcksum/state.c - ../3rdparty/zsync/c/libzsync/sha1.c - ../3rdparty/zsync/c/libzsync/zsync.c - ../3rdparty/zsync/c/libzsync/zsyncfile.c - ../3rdparty/zsync/c/progress.c -) - -set_target_properties(zsync PROPERTIES C_STANDARD 11 AUTOMOC OFF) -if (CMAKE_C_COMPILER_ID STREQUAL "GNU" OR CMAKE_C_COMPILER_ID MATCHES "Clang") - target_compile_options(zsync PRIVATE -Wno-unused-parameter -Wno-unused-function) -endif() -target_include_directories(zsync PUBLIC $<BUILD_INTERFACE:${CMAKE_SOURCE_DIR}/src/3rdparty/zsync/c>) - -if ( WIN32 ) - target_link_libraries(zsync PUBLIC ws2_32) - # ensure size_t is 64 bits - target_compile_definitions(zsync PRIVATE _FILE_OFFSET_BITS=64) -endif() - -target_compile_definitions(zsync PRIVATE VERSION="0.6.3" SIZEOF_OFF_T=8) - -## end zsync - # These headers are installed for libowncloudsync to be used by 3rd party apps set(owncloudsync_HEADERS account.h @@ -105,7 +74,6 @@ add_library(${synclib_NAME} SHARED ${libsync_SRCS}) target_link_libraries(${synclib_NAME} PUBLIC "${csync_NAME}" Qt5::Core Qt5::Network - zsync ) if ( APPLE ) diff --git a/src/libsync/capabilities.cpp b/src/libsync/capabilities.cpp index a4780ff6c..74b68fd15 100644 --- a/src/libsync/capabilities.cpp +++ b/src/libsync/capabilities.cpp @@ -213,11 +213,6 @@ bool Capabilities::versioningEnabled() const return _capabilities.value("files").toMap().value("versioning").toBool(); } -QString Capabilities::zsyncSupportedVersion() const -{ - return _capabilities[QStringLiteral("dav")].toMap()[QStringLiteral("zsync")].toString(); -} - QStringList Capabilities::blacklistedFiles() const { return _capabilities.value("files").toMap().value("blacklisted_files").toStringList(); diff --git a/src/libsync/capabilities.h b/src/libsync/capabilities.h index 5720805c3..dc26320cb 100644 --- a/src/libsync/capabilities.h +++ b/src/libsync/capabilities.h @@ -80,7 +80,6 @@ public: int defaultPermissions() const; bool chunkingNg() const; - QString zsyncSupportedVersion() const; /// Wheter to use chunking bool bigfilechunkingEnabled() const; diff --git a/src/libsync/configfile.cpp b/src/libsync/configfile.cpp index f4206f097..e4c015b01 100644 --- a/src/libsync/configfile.cpp +++ b/src/libsync/configfile.cpp @@ -89,9 +89,6 @@ static const char useNewBigFolderSizeLimitC[] = "useNewBigFolderSizeLimit"; static const char confirmExternalStorageC[] = "confirmExternalStorage"; static const char moveToTrashC[] = "moveToTrash"; -static const char deltaSyncEnabledC[] = "DeltaSync/enabled"; -static const char deltaSyncMinimumFileSizeC[] = "DeltaSync/minFileSize"; - const char certPath[] = "http_certificatePath"; const char certPasswd[] = "http_certificatePasswd"; QString ConfigFile::_confDir = QString(); @@ -734,28 +731,6 @@ void ConfigFile::setMoveToTrash(bool isChecked) setValue(moveToTrashC, isChecked); } -bool ConfigFile::deltaSyncEnabled() const -{ - QSettings settings(configFile(), QSettings::IniFormat); - return settings.value(QLatin1String(deltaSyncEnabledC), false).toBool(); // default to false -} - -void ConfigFile::setDeltaSyncEnabled(bool enabled) -{ - setValue(deltaSyncEnabledC, enabled); -} - -qint64 ConfigFile::deltaSyncMinFileSize() const -{ - QSettings settings(configFile(), QSettings::IniFormat); - return settings.value(QLatin1String(deltaSyncMinimumFileSizeC), 10 * 1024 * 1024).toLongLong(); // default to 10 MiB -} - -void ConfigFile::setDeltaSyncMinFileSize(qint64 bytes) -{ - setValue(deltaSyncMinimumFileSizeC, bytes); -} - bool ConfigFile::promptDeleteFiles() const { QSettings settings(configFile(), QSettings::IniFormat); diff --git a/src/libsync/configfile.h b/src/libsync/configfile.h index 5022a6580..185278b3a 100644 --- a/src/libsync/configfile.h +++ b/src/libsync/configfile.h @@ -136,12 +136,6 @@ public: void setNewBigFolderSizeLimit(bool isChecked, qint64 mbytes); bool confirmExternalStorage() const; void setConfirmExternalStorage(bool); - /** delta sync */ - bool deltaSyncEnabled() const; - void setDeltaSyncEnabled(bool enabled); - qint64 deltaSyncMinFileSize() const; // bytes - void setDeltaSyncMinFileSize(qint64 bytes); - /** If we should move the files deleted on the server in the trash */ bool moveToTrash() const; diff --git a/src/libsync/discoveryphase.cpp b/src/libsync/discoveryphase.cpp index 55dbd5c38..0f25648be 100644 --- a/src/libsync/discoveryphase.cpp +++ b/src/libsync/discoveryphase.cpp @@ -337,8 +337,7 @@ void DiscoverySingleDirectoryJob::start() << "http://owncloud.org/ns:downloadURL" << "http://owncloud.org/ns:dDC" << "http://owncloud.org/ns:permissions" - << "http://owncloud.org/ns:checksums" - << "http://owncloud.org/ns:zsync"; + << "http://owncloud.org/ns:checksums"; if (_isRootPath) props << "http://owncloud.org/ns:data-fingerprint"; if (_account->serverVersionInt() >= Account::makeServerVersion(10, 0, 0)) { @@ -406,14 +405,6 @@ static void propertyMapToRemoteInfo(const QMap<QString, QString> &map, RemoteInf // Piggy back on the persmission field result.remotePerm.setPermission(RemotePermissions::IsShared); } - } else if (property == "zsync" && value.toUtf8() == "true") { - // Since QMap is sorted, "zsync" is always after "permissions". - if (result.remotePerm.isNull()) { - qWarning() << "Server returned no permissions"; - // Empty permissions will cause a sync failure - } else { - result.remotePerm.setPermission(RemotePermissions::HasZSyncMetadata); - } } } } diff --git a/src/libsync/progressdispatcher.h b/src/libsync/progressdispatcher.h index 4bac0e339..daf3dd976 100644 --- a/src/libsync/progressdispatcher.h +++ b/src/libsync/progressdispatcher.h @@ -99,10 +99,6 @@ public: * This function is called at most once per item during propagation * to adjust them when its actual size has been determined. * - * Example: With delta-sync, the actual size of the download will only - * be known during propagation - this function adjusts the total size - * to account for it. - * * The value in item.size must be the same as during the call to * adjustTotalsForFile() while newSize is the newly determined actual * size. diff --git a/src/libsync/propagatecommonzsync.cpp b/src/libsync/propagatecommonzsync.cpp deleted file mode 100644 index 07a4d56a3..000000000 --- a/src/libsync/propagatecommonzsync.cpp +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright (C) by Ahmed Ammar <ahmed.a.ammar@gmail.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - */ - -extern "C" { -#include "libzsync/zsync.h" -#include "libzsync/zsyncfile.h" -} - -#include "config.h" -#include "propagateupload.h" -#include "owncloudpropagator_p.h" -#include "networkjobs.h" -#include "account.h" -#include "common/syncjournaldb.h" -#include "common/syncjournalfilerecord.h" -#include "common/utility.h" -#include "filesystem.h" -#include "propagatorjobs.h" -#include "syncengine.h" -#include "propagateremotemove.h" -#include "propagateremotedelete.h" -#include "common/asserts.h" - -#include <QNetworkAccessManager> -#include <QFileInfo> -#include <QDir> -#include <QTemporaryDir> - -#if defined(Q_CC_MSVC) -#include <io.h> // for dup -#endif - -namespace OCC { - -Q_LOGGING_CATEGORY(lcZsyncSeed, "sync.propagate.zsync.seed", QtInfoMsg) -Q_LOGGING_CATEGORY(lcZsyncGenerate, "sync.propagate.zsync.generate", QtInfoMsg) -Q_LOGGING_CATEGORY(lcZsyncGet, "sync.networkjob.zsync.get", QtInfoMsg) -Q_LOGGING_CATEGORY(lcZsyncPut, "sync.networkjob.zsync.put", QtInfoMsg) - -bool isZsyncPropagationEnabled(OwncloudPropagator *propagator, const SyncFileItemPtr &item) -{ - if (propagator->account()->capabilities().zsyncSupportedVersion() != "1.0") { - qCInfo(lcPropagator) << "[zsync disabled] Lack of server support."; - return false; - } - if (item->_remotePerm.hasPermission(RemotePermissions::IsMounted) || item->_remotePerm.hasPermission(RemotePermissions::IsMountedSub)) { - qCInfo(lcPropagator) << "[zsync disabled] External storage not supported."; - return false; - } - if (!propagator->syncOptions()._deltaSyncEnabled) { - qCInfo(lcPropagator) << "[zsync disabled] Client configuration option."; - return false; - } - if (item->_size < propagator->syncOptions()._deltaSyncMinFileSize) { - qCInfo(lcPropagator) << "[zsync disabled] File size is smaller than minimum."; - return false; - } - - return true; -} - -QUrl zsyncMetadataUrl(OwncloudPropagator *propagator, const QString &path) -{ - QUrlQuery urlQuery; - QList<QPair<QString, QString>> QueryItems({ { "zsync", nullptr } }); - urlQuery.setQueryItems(QueryItems); - return Utility::concatUrlPath(propagator->account()->davUrl(), propagator->_remoteFolder + path, urlQuery); -} - -void ZsyncSeedRunnable::run() -{ - // Create a temporary file to use with zsync_begin() - QTemporaryFile zsyncControlFile; - zsyncControlFile.open(); - zsyncControlFile.write(_zsyncData.constData(), _zsyncData.size()); - zsyncControlFile.flush(); - - int fileHandle = zsyncControlFile.handle(); - zsync_unique_ptr<FILE> f(fdopen(dup(fileHandle), "r"), [](FILE *f) { - fclose(f); - }); - zsyncControlFile.close(); - rewind(f.get()); - - QByteArray tmp_file; - if (!_tmpFilePath.isEmpty()) { - tmp_file = _tmpFilePath.toLocal8Bit(); - } else { - QTemporaryFile tmpFile; - tmpFile.open(); - tmp_file = tmpFile.fileName().toLocal8Bit(); - tmpFile.close(); - } - - const char *tfname = tmp_file; - - zsync_unique_ptr<struct zsync_state> zs(zsync_begin(f.get(), tfname), - [](struct zsync_state *zs) { zsync_end(zs); }); - if (!zs) { - QString errorString = tr("Unable to parse zsync file."); - emit failedSignal(errorString); - return; - } - - { - // Open the content input file. - // - // Zsync expects a FILE*. We get that by opening a QFile, getting the file - // descriptor with ::handle() and then duping and fdopening that. - // - // A less convoluted alternative would be to just get the file descriptor - // - FileSystem::openAndSeekFileSharedRead() already has it on windows - - // and to skip the QFile::open() entirely. - QFile file(_zsyncFilePath); - QString error; - if (!FileSystem::openAndSeekFileSharedRead(&file, &error, 0)) { - QString errorString = tr("Unable to open file: %1").arg(error); - emit failedSignal(errorString); - return; - } - - /* Give the contents to libzsync to read, to find any content that - * is part of the target file. */ - qCInfo(lcZsyncSeed) << "Reading seed file:" << _zsyncFilePath; - int fileHandle = file.handle(); - zsync_unique_ptr<FILE> f(fdopen(dup(fileHandle), "r"), [](FILE *f) { - fclose(f); - }); - file.close(); - rewind(f.get()); - zsync_submit_source_file(zs.get(), f.get(), false, _type == ZsyncMode::download ? false : true); - } - - emit finishedSignal(zs.release()); -} - -static void log_zsync_errors(const char *func, FILE *stream, void */*error_context*/) -{ - qCWarning(lcZsyncGenerate) << "Zsync error: " << func << ": " << strerror(ferror(stream)); -} - -void ZsyncGenerateRunnable::run() -{ - // Create a temporary file to use with zsync_begin() - QTemporaryFile zsynctf, zsyncmeta; - zsyncmeta.open(); - zsynctf.open(); - - int metaHandle = zsyncmeta.handle(); - zsync_unique_ptr<FILE> meta(fdopen(dup(metaHandle), "w"), [](FILE *f) { - fclose(f); - }); - zsyncmeta.close(); - - int tfHandle = zsynctf.handle(); - zsync_unique_ptr<FILE> tf(fdopen(dup(tfHandle), "w+"), [](FILE *f) { - fclose(f); - }); - zsynctf.close(); - - /* Ensure that metadata file is not buffered, since we are using handles directly */ - setvbuf(meta.get(), nullptr, _IONBF, 0); - - qCDebug(lcZsyncGenerate) << "Starting generation of:" << _file; - - // See ZsyncSeedRunnable for details on FILE creation - QFile inFile(_file); - QString error; - if (!FileSystem::openAndSeekFileSharedRead(&inFile, &error, 0)) { - FileSystem::remove(zsyncmeta.fileName()); - QString error = tr("Failed to open input file %1: %2").arg(_file, error); - emit failedSignal(error); - return; - } - zsync_unique_ptr<FILE> in(fdopen(dup(inFile.handle()), "r"), [](FILE *f) { - fclose(f); - }); - if (!in) { - FileSystem::remove(zsyncmeta.fileName()); - QString error = tr("Failed to open input file: %1").arg(_file); - emit failedSignal(error); - return; - } - - zsync_unique_ptr<zsyncfile_state> state(zsyncfile_init(ZSYNC_BLOCKSIZE), [](zsyncfile_state *state) { - zsyncfile_finish(&state); - }); - state->stream_error = &log_zsync_errors; - - /* Read the input file and construct the checksum of the whole file, and - * the per-block checksums */ - if (zsyncfile_read_stream_write_blocksums(in.get(), tf.get(), /*no_look_inside=*/1, state.get()) != 0) { - QString error = QString(tr("Failed to write block sums:")) + _file; - emit failedSignal(error); - return; - } - - // We don't care for the optimal checksum lengths computed by - // compute_rsum_checksum_len() since we use blocks much larger - // than the default (1 MB instead of 8 kB) and can just store the full - // 24 bytes per block. - int rsum_len = 8; - int checksum_len = 16; - - if (zsyncfile_write( - meta.get(), tf.get(), - rsum_len, checksum_len, - 0, nullptr, nullptr, // recompress - nullptr, 0, // fname, mtime - nullptr, 0, // urls - nullptr, 0, // Uurls - state.get()) - != 0) { - QString error = QString(tr("Failed to write zsync metadata file:")) + _file; - emit failedSignal(error); - return; - } - - qCDebug(lcZsyncGenerate) << "Done generation of:" << zsyncmeta.fileName(); - - zsyncmeta.setAutoRemove(false); - emit finishedSignal(zsyncmeta.fileName()); -} -} diff --git a/src/libsync/propagatecommonzsync.h b/src/libsync/propagatecommonzsync.h deleted file mode 100644 index b647694f9..000000000 --- a/src/libsync/propagatecommonzsync.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) by Ahmed Ammar <ahmed.a.ammar@gmail.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - */ - -#pragma once - -#include <QLoggingCategory> -#include <QTemporaryFile> -#include <QRunnable> -#include <QThreadPool> - -#define ZSYNC_BLOCKSIZE (1 * 1024 * 1024) // must be power of 2 - -namespace OCC { -Q_DECLARE_LOGGING_CATEGORY(lcZsyncPut) -Q_DECLARE_LOGGING_CATEGORY(lcZsyncGet) - -enum class ZsyncMode { download, - upload }; - -template <typename T> -using zsync_unique_ptr = std::unique_ptr<T, std::function<void(T *)>>; - -/** - * @ingroup libsync - * - * Helper function to know if we are allowed to attempt using zsync from configuration/command-line options. - * - */ -bool isZsyncPropagationEnabled(OwncloudPropagator *propagator, const SyncFileItemPtr &item); - -/** - * @ingroup libsync - * - * Helper function to get zsync metadata Url. - * - */ -QUrl zsyncMetadataUrl(OwncloudPropagator *propagator, const QString &path); - -/** - * @ingroup libsync - * - * Helper runnable to 'seed' the zsync_state by providing the downloaded metadata and seed file. - * This is needed for both upload and download since they both must seed the zsync_state to know which - * ranges to upload/download. - * - */ -class ZsyncSeedRunnable : public QObject, public QRunnable -{ - Q_OBJECT - QByteArray _zsyncData; - QString _zsyncFilePath; - QString _tmpFilePath; - ZsyncMode _type; - -public: - explicit ZsyncSeedRunnable(QByteArray &zsyncData, QString path, ZsyncMode type, QString tmpFilePath = nullptr) - : _zsyncData(zsyncData) - , _zsyncFilePath(path) - , _tmpFilePath(tmpFilePath) - , _type(type){}; - - void run() override; - -signals: - void finishedSignal(void *zs); - void failedSignal(const QString &errorString); -}; - -/** - * @ingroup libsync - * - * Helper runnable to generate zsync metadata file when uploading. - * Takes an input file path and returns a zsync metadata file path finsihed. - * - */ -class ZsyncGenerateRunnable : public QObject, public QRunnable -{ - Q_OBJECT - const QString _file; - -public: - explicit ZsyncGenerateRunnable(const QString &file) - : _file(file){}; - - void run() override; - -signals: - void finishedSignal(const QString &generatedFileName); - void failedSignal(const QString &errorString); -}; -} diff --git a/src/libsync/propagatedownload.cpp b/src/libsync/propagatedownload.cpp index c6aa98eac..0e9435c91 100644 --- a/src/libsync/propagatedownload.cpp +++ b/src/libsync/propagatedownload.cpp @@ -524,51 +524,9 @@ void PropagateDownloadFile::startDownload() propagator()->_journal->commit("download file start"); } - if (_item->_remotePerm.hasPermission(RemotePermissions::HasZSyncMetadata) && isZsyncPropagationEnabled(propagator(), _item)) { - if (_item->_previousSize) { - // Retrieve zsync metadata file from the server - qCInfo(lcZsyncGet) << "Retrieving zsync metadata for:" << _item->_file; - QNetworkRequest req; - req.setPriority(QNetworkRequest::LowPriority); - QUrl zsyncUrl = zsyncMetadataUrl(propagator(), _item->_file); - auto job = propagator()->account()->sendRequest("GET", zsyncUrl, req); - connect(job, &SimpleNetworkJob::finishedSignal, this, &PropagateDownloadFile::slotZsyncGetMetaFinished); - return; - } - qCInfo(lcZsyncGet) << "No local copy of:" << _item->_file; - } - startFullDownload(); } - -void PropagateDownloadFile::slotZsyncGetMetaFinished(QNetworkReply *reply) -{ - int httpStatusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (httpStatusCode / 100 != 2) { - /* Fall back to full download */ - qCWarning(lcZsyncGet) << "Failed to retrieve zsync metadata for:" << _item->_file; - startFullDownload(); - return; - } - - QByteArray zsyncData = reply->readAll(); - _expectedEtagForResume = getEtagFromReply(reply); - qCInfo(lcZsyncGet) << "Retrieved zsync metadata for:" << _item->_file << "size:" << zsyncData.size() - << "etag:" << _expectedEtagForResume; - - QMap<QByteArray, QByteArray> headers; - _job = new GETFileZsyncJob(propagator(), _item, propagator()->_remoteFolder + _item->_file, - &_tmpFile, headers, _expectedEtagForResume, zsyncData, this); - connect(_job.data(), &GETJob::finishedSignal, this, &PropagateDownloadFile::slotGetFinished); - connect(qobject_cast<GETFileZsyncJob *>(_job.data()), &GETFileZsyncJob::overallDownloadProgress, - this, &PropagateDownloadFile::slotDownloadProgress); - _job->setBandwidthManager(&propagator()->_bandwidthManager); - propagator()->_activeJobList.append(this); - _job->start(); - _isDeltaSyncDownload = true; -} - void PropagateDownloadFile::startFullDownload() { QMap<QByteArray, QByteArray> headers; @@ -620,28 +578,6 @@ void PropagateDownloadFile::slotGetFinished() GETJob *job = _job; OC_ASSERT(job); - SyncFileItem::Status status = job->errorStatus(); - - // Needed because GETFileZsyncJob may emit finishedSignal without any further network activity - if (!job->reply()) { - if (status == SyncFileItem::Success) { - _tmpFile.close(); - _tmpFile.flush(); - downloadFinished(); - return; - } - - FileSystem::remove(_tmpFile.fileName()); - if (status != SyncFileItem::NoStatus) { - done(status, job->errorString()); - return; - } - - OC_ASSERT_X(false, "Download slot finished, but there was no reply!"); - done(SyncFileItem::FatalError, tr("Download slot finished, but there was no reply!")); - return; - } - _item->_httpErrorCode = job->reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); _item->_responseTimeStamp = job->responseTimestamp(); _item->_requestId = job->requestId(); @@ -746,7 +682,7 @@ void PropagateDownloadFile::slotGetFinished() hasSizeHeader = false; } - if (!_isDeltaSyncDownload && hasSizeHeader && _tmpFile.size() > 0 && bodySize == 0) { + if (hasSizeHeader && _tmpFile.size() > 0 && bodySize == 0) { // Strange bug with broken webserver or webfirewall https://github.com/owncloud/client/issues/3373#issuecomment-122672322 // This happened when trying to resume a file. The Content-Range header was files, Content-Length was == 0 qCDebug(lcPropagateDownload) << bodySize << _item->_size << _tmpFile.size() << job->resumeStart(); @@ -755,7 +691,7 @@ void PropagateDownloadFile::slotGetFinished() return; } - if (!_isDeltaSyncDownload && bodySize > 0 && bodySize != _tmpFile.size() - job->resumeStart()) { + if (bodySize > 0 && bodySize != _tmpFile.size() - job->resumeStart()) { qCDebug(lcPropagateDownload) << bodySize << _tmpFile.size() << job->resumeStart(); propagator()->_anotherSyncNeeded = true; done(SyncFileItem::SoftError, tr("The file could not be downloaded completely.")); diff --git a/src/libsync/propagatedownload.h b/src/libsync/propagatedownload.h index efa5dd782..3f01ba06c 100644 --- a/src/libsync/propagatedownload.h +++ b/src/libsync/propagatedownload.h @@ -15,7 +15,6 @@ #include "owncloudpropagator.h" #include "networkjobs.h" -#include "propagatecommonzsync.h" #include <QBuffer> #include <QFile> @@ -69,67 +68,6 @@ signals: }; /** - * @brief Downloads the zsync metadata and uses the original file as a seed, then downloads needed ranges via GET - * @ingroup libsync - */ -class GETFileZsyncJob : public GETJob -{ - Q_OBJECT - QFile *_device; - SyncFileItemPtr _item; - OwncloudPropagator *_propagator; - QMap<QByteArray, QByteArray> _headers; - QByteArray _expectedEtagForResume; - bool _hasEmittedFinishedSignal; - QByteArray _zsyncData; - int _nrange = 0; - int _current = 0; - off_t _pos = 0; - off_t _received = 0; - /* these must be in this order so the destructors are done in the right order */ - zsync_unique_ptr<struct zsync_state> _zs = nullptr; - zsync_unique_ptr<struct zsync_receiver> _zr = nullptr; - - /** Byte ranges that need to be received. - * - * As returned by zsync_needed_byte_ranges() - * - * That means: [begin0, end0, begin1, end1, ...] - * where begin and end are *inclusive* and the last - * end may very well exceed the total file size. - */ - zsync_unique_ptr<off_t> _zbyterange = nullptr; - -public: - // DOES NOT take ownership of the device. - GETFileZsyncJob(OwncloudPropagator *propagator, SyncFileItemPtr &item, const QString &path, QFile *device, - const QMap<QByteArray, QByteArray> &headers, const QByteArray &expectedEtagForResume, - const QByteArray &zsyncData, QObject *parent = nullptr); - - qint64 currentDownloadPosition() override; - - void start() override; - bool finished() override; - -private: - void seedFinished(void *zs); - void seedFailed(const QString &errorString); - - void startCurrentRange(qint64 start = 0, qint64 end = 0); - -private slots: - void slotReadyRead(); - void slotMetaDataChanged(); - -public slots: - void slotOverallDownloadProgress(qint64, qint64); - -signals: - void overallDownloadProgress(qint64, qint64); -}; - - -/** * @brief Downloads the remote file via GET * @ingroup libsync */ @@ -210,36 +148,29 @@ signals: | + | checksum differs? | +-> startDownload() <--------------------------------------------+ - + | - +-> isZsyncPropagationEnabled()? | - + | - +-+ yes +> local file exists? | - | + | - | +-+ yes +------> run a GETFIleZsyncJob | - | | | - + + done? +------------+ | - no no | | - + + | | - | v | | - +-> startFullDownload() | | - + | | - +-> run a GETFileJob | | checksum identical? - | | - done?+> slotGetFinished() <--------+ | - + | - +-> validate checksum header | + + + | | + no no | | + + + | | + | v | | + +-> startFullDownload() | | + + | | + +-> run a GETFileJob | | checksum identical? + | | + done?+> slotGetFinished() <--------+ | + + | + +-> validate checksum header | | - done?+> transmissionChecksumValidated() | - + | - +-> compute the content checksum | + done?+> transmissionChecksumValidated() | + + | + +-> compute the content checksum | | - done?+> contentChecksumComputed() | - + | - +-> downloadFinished() | - + | - +------------------+ | - | | - +-> updateMetadata() <-------------------------------+ + done?+> contentChecksumComputed() | + + | + +-> downloadFinished() | + + | + +------------------+ | + | | + +-> updateMetadata() <---------------------------------------+ \endcode */ @@ -247,7 +178,6 @@ class PropagateDownloadFile : public PropagateItemJob { Q_OBJECT QByteArray _expectedEtagForResume; - bool _isDeltaSyncDownload = false; public: PropagateDownloadFile(OwncloudPropagator *propagator, const SyncFileItemPtr &item) @@ -283,8 +213,6 @@ private slots: void startFullDownload(); /// Called when the GETJob finishes void slotGetFinished(); - /// Called when the we have finished getting the zsync metadata file - void slotZsyncGetMetaFinished(QNetworkReply *reply); /// Called when the download's checksum header was validated void transmissionChecksumValidated(const QByteArray &checksumType, const QByteArray &checksum); /// Called when the download's checksum computation is done diff --git a/src/libsync/propagatedownloadzsync.cpp b/src/libsync/propagatedownloadzsync.cpp deleted file mode 100644 index 1a6c0300e..000000000 --- a/src/libsync/propagatedownloadzsync.cpp +++ /dev/null @@ -1,335 +0,0 @@ -/* - * Copyright (C) by Ahmed Ammar <ahmed.a.ammar@gmail.com> - * - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation the - * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to - * permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the - * Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - * WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - * COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - * - */ - -extern "C" { -#include "libzsync/zsync.h" -} - -#include "config.h" -#include "owncloudpropagator_p.h" -#include "propagatedownload.h" -#include "networkjobs.h" -#include "account.h" -#include "common/syncjournaldb.h" -#include "common/syncjournalfilerecord.h" -#include "common/utility.h" -#include "filesystem.h" -#include "propagatorjobs.h" -#include "propagateremotedelete.h" -#include "common/checksums.h" -#include "common/asserts.h" - -#include <QLoggingCategory> -#include <QNetworkAccessManager> -#include <QFileInfo> -#include <QDir> -#include <QTemporaryFile> -#include <cmath> - -#ifdef Q_OS_UNIX -#include <unistd.h> -#endif - -namespace OCC { - -// DOES NOT take ownership of the device. -GETFileZsyncJob::GETFileZsyncJob(OwncloudPropagator *propagator, SyncFileItemPtr &item, - const QString &path, QFile *device, const QMap<QByteArray, QByteArray> &headers, - const QByteArray &expectedEtagForResume, const QByteArray &zsyncData, QObject *parent) - : GETJob(propagator->account(), path, parent) - , _device(device) - , _item(item) - , _propagator(propagator) - , _headers(headers) - , _expectedEtagForResume(expectedEtagForResume) - , _hasEmittedFinishedSignal(false) - , _zsyncData(zsyncData) -{ -} - -void GETFileZsyncJob::startCurrentRange(qint64 start, qint64 end) -{ - // The end of the range might exceed the file size. - // It's size-1 because the Range header is end-inclusive. - end = qMin(end, _item->_size - 1); - - _headers["Range"] = "bytes=" + QByteArray::number(start) + '-' + QByteArray::number(end); - - qCDebug(lcZsyncGet) << path() << "HTTP GET with range" << _headers["Range"]; - - QNetworkRequest req; - for (QMap<QByteArray, QByteArray>::const_iterator it = _headers.begin(); it != _headers.end(); ++it) { - req.setRawHeader(it.key(), it.value()); - } - req.setPriority(QNetworkRequest::LowPriority); // Long downloads must not block non-propagation jobs. - - sendRequest("GET", makeDavUrl(path()), req); - - reply()->setReadBufferSize(16 * 1024); // keep low so we can easier limit the bandwidth - qCDebug(lcZsyncGet) << _bandwidthManager << _bandwidthChoked << _bandwidthLimited; - if (_bandwidthManager) { - _bandwidthManager->registerDownloadJob(this); - } - - if (reply()->error() != QNetworkReply::NoError) { - qCWarning(lcZsyncGet) << " Network error: " << errorString(); - } - - _pos = 0; - - connect(reply(), &QNetworkReply::downloadProgress, this, &GETFileZsyncJob::slotOverallDownloadProgress); - connect(reply(), &QIODevice::readyRead, this, &GETFileZsyncJob::slotReadyRead); - connect(reply(), &QNetworkReply::metaDataChanged, this, &GETFileZsyncJob::slotMetaDataChanged); - connect(this, &AbstractNetworkJob::networkActivity, account().data(), &Account::propagatorNetworkActivity); - - AbstractNetworkJob::start(); -} - -bool GETFileZsyncJob::finished() -{ - if (reply()->bytesAvailable()) { - return false; - } - - // zsync_receive_data will only complete once we have sent block aligned data - off_t range_size = _zbyterange.get()[(2 * _current) + 1] - _zbyterange.get()[(2 * _current)] + 1; - if (_pos < range_size) { - QByteArray fill(range_size - _pos, 0); - qCDebug(lcZsyncGet) << "About to zsync" << range_size - _pos << "filler bytes @" << _zbyterange.get()[2 * _current] << "pos:" << _pos << "of" << path(); - if (zsync_receive_data(_zr.get(), (const unsigned char *)fill.constData(), _zbyterange.get()[2 * _current] + _pos, range_size - _pos) != 0) { - _errorString = "Failed to receive data for: " + _propagator->getFilePath(_item->_file); - _errorStatus = SyncFileItem::NormalError; - qCWarning(lcZsyncGet) << "Error while writing to file:" << _errorString; - reply()->abort(); - emit finishedSignal(); - return true; - } - } - - // chain the next range if we still have some - if (_current < _nrange - 1) { - _current++; - startCurrentRange(_zbyterange.get()[2 * _current], _zbyterange.get()[(2 * _current) + 1]); - return false; - } - - if (!_hasEmittedFinishedSignal) { - _zr.reset(); - _zs.reset(); // ensure the file is closed. - emit finishedSignal(); - } - - _hasEmittedFinishedSignal = true; - - return true; // discard -} - -void GETFileZsyncJob::seedFinished(void *zs) -{ - _zs = zsync_unique_ptr<struct zsync_state>(static_cast<struct zsync_state *>(zs), [](struct zsync_state *zs) { - zsync_complete(zs); - zsync_end(zs); - }); - if (!_zs) { - _errorString = tr("Unable to parse zsync."); - _errorStatus = SyncFileItem::NormalError; - qCDebug(lcZsyncGet) << _errorString; - emit finishedSignal(); - return; - } - - { /* And print how far we've progressed towards the target file */ - long long done, total; - - zsync_progress(_zs.get(), &done, &total); - qCInfo(lcZsyncGet).nospace() << "Done reading: " - << _propagator->getFilePath(_item->_file) - << " " << fixed << qSetRealNumberPrecision(1) << (100.0f * done) / total - << "% of target seeded."; - } - - /* Get a set of byte ranges that we need to complete the target */ - _zbyterange = zsync_unique_ptr<off_t>(zsync_needed_byte_ranges(_zs.get(), &_nrange, 0), [](off_t *zbr) { - free(zbr); - }); - if (!_zbyterange) { - _errorString = tr("Failed to get zsync byte ranges."); - _errorStatus = SyncFileItem::NormalError; - qCDebug(lcZsyncGet) << _errorString; - emit finishedSignal(); - return; - } - - qCDebug(lcZsyncGet) << "Number of ranges:" << _nrange; - - /* If we have no ranges then we have equal files and we are done */ - if (_nrange == 0 && _item->_size == qint64(zsync_file_length(_zs.get()))) { - _propagator->reportFileTotal(*_item, 0); - _errorStatus = SyncFileItem::Success; - _zr.reset(); - _zs.reset(); // ensure the file is closed. - emit finishedSignal(); - return; - } - - _zr = zsync_unique_ptr<struct zsync_receiver>(zsync_begin_receive(_zs.get(), 0), [](struct zsync_receiver *zr) { - zsync_end_receive(zr); - }); - if (!_zr) { - _errorString = tr("Failed to initialize zsync receive structure."); - _errorStatus = SyncFileItem::NormalError; - qCDebug(lcZsyncGet) << _errorString; - emit finishedSignal(); - return; - } - - quint64 totalBytes = 0; - for (int i = 0; i < _nrange; i++) { - totalBytes += _zbyterange.get()[(2 * i) + 1] - _zbyterange.get()[(2 * i)] + 1; - } - - qCDebug(lcZsyncGet) << "Total bytes:" << totalBytes; - _propagator->reportFileTotal(*_item, totalBytes); - - /* start getting bytes for first zsync byte range */ - startCurrentRange(_zbyterange.get()[0], _zbyterange.get()[1]); -} - -void GETFileZsyncJob::seedFailed(const QString &errorString) -{ - _errorString = errorString; - _errorStatus = SyncFileItem::NormalError; - - qCCritical(lcZsyncGet) << _errorString; - - /* delete remote zsync file */ - QUrl zsyncUrl = zsyncMetadataUrl(_propagator, _item->_file); - (new DeleteJob(_propagator->account(), zsyncUrl, this))->start(); - - emit finishedSignal(); -} - -void GETFileZsyncJob::start() -{ - ZsyncSeedRunnable *run = new ZsyncSeedRunnable(_zsyncData, _propagator->getFilePath(_item->_file), - ZsyncMode::download, _device->fileName()); - connect(run, &ZsyncSeedRunnable::finishedSignal, this, &GETFileZsyncJob::seedFinished); - connect(run, &ZsyncSeedRunnable::failedSignal, this, &GETFileZsyncJob::seedFailed); - - // Starts in a seperate thread - QThreadPool::globalInstance()->start(run); -} - -qint64 GETFileZsyncJob::currentDownloadPosition() -{ - return _received; -} - -void GETFileZsyncJob::slotReadyRead() -{ - if (!reply()) - return; - - int bufferSize = qMin(1024 * 8ll, reply()->bytesAvailable()); - QByteArray buffer(bufferSize, Qt::Uninitialized); - - while (reply()->bytesAvailable() > 0) { - if (_bandwidthChoked) { - qCWarning(lcZsyncGet) << "Download choked"; - return; - } - qint64 toRead = bufferSize; - if (_bandwidthLimited) { - toRead = qMin(qint64(bufferSize), _bandwidthQuota); - if (toRead == 0) { - qCWarning(lcZsyncGet) << "Out of quota"; - return; - } - _bandwidthQuota -= toRead; - } - - qint64 r = reply()->read(buffer.data(), toRead); - if (r < 0) { - _errorString = networkReplyErrorString(*reply()); - _errorStatus = SyncFileItem::NormalError; - qCWarning(lcZsyncGet) << "Error while reading from device: " << _errorString; - reply()->abort(); - return; - } - - if (!_nrange) { - qCWarning(lcZsyncGet) << "No ranges to fetch."; - _received += r; - _pos += r; - return; - } - - qCDebug(lcZsyncGet) << "About to zsync" << r << "bytes @" << _zbyterange.get()[2 * _current] << "pos:" << _pos << "of" << path(); - - if (zsync_receive_data(_zr.get(), (const unsigned char *)buffer.constData(), _zbyterange.get()[2 * _current] + _pos, r) != 0) { - _errorString = "Failed to receive data for: " + _propagator->getFilePath(_item->_file); - _errorStatus = SyncFileItem::NormalError; - qCWarning(lcZsyncGet) << "Error while writing to file:" << _errorString; - reply()->abort(); - return; - } - - _received += r; - _pos += r; - } -} - -void GETFileZsyncJob::slotMetaDataChanged() -{ - // For some reason setting the read buffer in GETFileJob::start doesn't seem to go - // through the HTTP layer thread(?) - reply()->setReadBufferSize(16 * 1024); - - int httpStatus = reply()->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - - // If the status code isn't 2xx, don't write the reply body to the file. - // For any error: handle it when the job is finished, not here. - if (httpStatus / 100 != 2) { - _device->close(); - return; - } - if (reply()->error() != QNetworkReply::NoError) { - return; - } - _etag = getEtagFromReply(reply()); - - if (!_expectedEtagForResume.isEmpty() && _expectedEtagForResume != _etag) { - qCWarning(lcZsyncGet) << "We received a different E-Tag for delta!" - << _expectedEtagForResume << "vs" << _etag; - _errorString = tr("We received a different E-Tag for delta. Retrying next time."); - _errorStatus = SyncFileItem::NormalError; - reply()->abort(); - return; - } - - auto lastModified = reply()->header(QNetworkRequest::LastModifiedHeader); - if (!lastModified.isNull()) { - _lastModified = Utility::qDateTimeToTime_t(lastModified.toDateTime()); - } -} - -void GETFileZsyncJob::slotOverallDownloadProgress(qint64, qint64) -{ - emit overallDownloadProgress(_received, 0); -} -} diff --git a/src/libsync/propagateupload.h b/src/libsync/propagateupload.h index 3ee58070b..964d0dc88 100644 --- a/src/libsync/propagateupload.h +++ b/src/libsync/propagateupload.h @@ -15,7 +15,6 @@ #include "owncloudpropagator.h" #include "networkjobs.h" -#include "propagatecommonzsync.h" #include <QBuffer> #include <QFile> @@ -361,8 +360,7 @@ private: /** Amount of data that needs to be sent to the server in bytes. * - * For normal uploads this will be the file size, for zsync uploads it can - * be less. + * For normal uploads this will be the file size. * * This value is intended to be comparable to _sent: it's always the total * amount of data that needs to be present at the server to finish the upload - @@ -374,8 +372,6 @@ private: qint64 _currentChunkOffset = 0; /// byte offset of the next chunk data that will be sent qint64 _currentChunkSize = 0; /// current chunk size bool _removeJobError = false; /// if not null, there was an error removing the job - bool _zsyncSupported = false; /// if zsync is supported this will be set to true - bool _isZsyncMetadataUploadRunning = false; // flag to ensure that zsync metadata upload is complete before job is // Map chunk number with its size from the PROPFIND on resume. // (Only used from slotPropfindIterate/slotPropfindFinished because the LsColJob use signals to report data.) @@ -432,12 +428,6 @@ private slots: void slotDeleteJobFinished(); void slotMkColFinished(QNetworkReply::NetworkError); void slotPutFinished(); - void slotZsyncGetMetaFinished(QNetworkReply *reply); - void slotZsyncSeedFinished(void *zs); - void slotZsyncSeedFailed(const QString &errorString); - void slotZsyncGenerationFinished(const QString &fileName); - void slotZsyncGenerationFailed(const QString &errorString); - void slotZsyncMetadataUploadFinished(); void slotMoveJobFinished(); void slotUploadProgress(qint64, qint64); }; diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index 25434918c..42d73953f 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -12,10 +12,6 @@ * for more details. */ -extern "C" { -#include "libzsync/zsync.h" -} - #include "config.h" #include "propagateupload.h" #include "owncloudpropagator_p.h" @@ -53,146 +49,30 @@ QUrl PropagateUploadFileNG::chunkUrl(qint64 chunkOffset) return Utility::concatUrlPath(propagator()->account()->url(), path); } -void PropagateUploadFileNG::slotZsyncSeedFinished(void *_zs) -{ - zsync_unique_ptr<struct zsync_state> zs(static_cast<struct zsync_state *>(_zs), [](struct zsync_state *zs) { - zsync_end(zs); - }); - { /* And print how far we've progressed towards the target file */ - long long done, total; - - zsync_progress(zs.get(), &done, &total); - qCInfo(lcZsyncPut).nospace() << "Done reading: " - << _item->_file << " " << fixed << qSetRealNumberPrecision(1) << (100.0f * done) / total - << "% of target seeded."; - } - - /* Get a set of byte ranges that we need to complete the target */ - int _nrange = 0; - zsync_unique_ptr<off_t> zbyterange(zsync_needed_byte_ranges(zs.get(), &_nrange, 0), [](off_t *zbr) { - free(zbr); - }); - if (!zbyterange) { - abortWithError(SyncFileItem::NormalError, tr("Failed to get zsync byte ranges.")); - return; - } - - qCDebug(lcZsyncPut) << "Number of ranges:" << _nrange; - - // The remote file size according to zsync metadata - auto remoteSize = static_cast<qint64>(zsync_file_length(zs.get())); - - /* If we have no ranges then we have equal files and we are done */ - if (_nrange == 0 && _item->_size == remoteSize) { - propagator()->reportFileTotal(*_item, 0); - finalize(); - return; - } - - // Size of combined uploads - int totalBytes = 0; - - /* The rightmost range returned by zsync can be larger than the local or remote - * file size. That happens because zsync only considers whole blocks. - * - * There are two symmetric cases: - * - * local smaller than remote (block size 6 bytes) - * local: abcdefg - * remote: abcdefaaa - * blocks: 000000 - * desired: 0 - * - * remote smaller than local (block size 6 bytes) - * local: abcdefaaa - * remote: abcdefg - * blocks: 000000 - * desired: 011 (so two blocks, the second one will be added below) - * - * To address this we limit the zsync blocks to min(localsize, remotesize). That - * way we only attempt to upload data that's locally available (local smaller) - * and can easily handle the case of a locally grown file below (remote smaller). - */ - qint64 minSize = qMin(_item->_size, remoteSize); - for (int i = 0; i < _nrange; i++) { - UploadRangeInfo rangeinfo = { qint64(zbyterange.get()[(2 * i)]), qint64(zbyterange.get()[(2 * i) + 1]) - qint64(zbyterange.get()[(2 * i)]) + 1 }; - if (rangeinfo.start < minSize) { - if (rangeinfo.end() > minSize) - rangeinfo.size = minSize - rangeinfo.start; - _rangesToUpload.append(rangeinfo); - totalBytes += rangeinfo.size; - } - } - - // If the local file has grown, upload the new local data - if (_item->_size > remoteSize) { - qint64 appendedBytes = _item->_size - remoteSize; - // Append to the last range if possible - if (!_rangesToUpload.isEmpty() && _rangesToUpload.last().end() == remoteSize) { - _rangesToUpload.last().size += appendedBytes; - } else { - UploadRangeInfo rangeinfo = { remoteSize, appendedBytes }; - _rangesToUpload.append(rangeinfo); - } - totalBytes += appendedBytes; - } - - for (const auto &range : _rangesToUpload) - qCDebug(lcZsyncPut) << "Upload range:" << range.start << range.size; - qCDebug(lcZsyncPut) << "Total bytes:" << totalBytes << "of file size" << _item->_size; - - propagator()->reportFileTotal(*_item, totalBytes); - _bytesToUpload = totalBytes; - - doStartUploadNext(); -} - -void PropagateUploadFileNG::slotZsyncSeedFailed(const QString &errorString) -{ - qCCritical(lcZsyncPut) << errorString; - - /* delete remote zsync file */ - QUrl zsyncUrl = zsyncMetadataUrl(propagator(), _item->_file); - (new DeleteJob(propagator()->account(), zsyncUrl, this))->start(); - - abortWithError(SyncFileItem::NormalError, errorString); -} /* State machine: +---> doStartUpload() - isZsyncPropagationEnabled()? +--+ yes +---> Download and seed zsync metadata and set-up new _rangesToUpload - + + - |no | - | | - | | - +^--------------------------------------------------------------+ - v doStartUploadNext() - isZsyncPropagationEnabled()? +--+ yes +---> Generate new zsync metadata file +--------------------+ - + + | - |no | | - | | Upload .zsync chunk - v | | - Check the db: is there an entry? <-------------------------+ | - + + | - |no |yes | - | v | - v PROPFIND | - startNewUpload() <-+ +-------------------------------------+ | - + | + | | - MKCOL + slotPropfindFinishedWithError() slotPropfindFinished() | - + Is there stale files to remove? | - slotMkColFinished() + + | - + no yes | - | + + | - | | DeleteJob | - | | + | - +-----+^------------------------------------------------------+^--+ slotDeleteJobFinished() | - | | - | +--------------------------------------------------------------------+ - | v + Check the db: is there an entry? + + + + |no |yes + | v + v PROPFIND + startNewUpload() <-+ +-------------------------------------+ + + | + | + MKCOL + slotPropfindFinishedWithError() slotPropfindFinished() + + Is there stale files to remove? + slotMkColFinished() + + + + no yes + | + + + | | DeleteJob + | | + + +-----+^------------------------------------------------------+^--+ slotDeleteJobFinished() + | + | + | +----> startNextChunk() +-> finished? +- ^ + | +---------------+ | @@ -206,63 +86,15 @@ void PropagateUploadFileNG::doStartUpload() { propagator()->_activeJobList.append(this); - _zsyncSupported = isZsyncPropagationEnabled(propagator(), _item); - if (_zsyncSupported && _item->_remotePerm.hasPermission(RemotePermissions::HasZSyncMetadata)) { - // Retrieve zsync metadata file from the server - qCInfo(lcZsyncPut) << "Retrieving zsync metadata for:" << _item->_file; - QNetworkRequest req; - req.setPriority(QNetworkRequest::LowPriority); - QUrl zsyncUrl = zsyncMetadataUrl(propagator(), _item->_file); - auto job = propagator()->account()->sendRequest("GET", zsyncUrl, req); - connect(job, &SimpleNetworkJob::finishedSignal, this, &PropagateUploadFileNG::slotZsyncGetMetaFinished); - return; - } - UploadRangeInfo rangeinfo = { 0, _item->_size }; _rangesToUpload.append(rangeinfo); _bytesToUpload = _item->_size; doStartUploadNext(); } -void PropagateUploadFileNG::slotZsyncGetMetaFinished(QNetworkReply *reply) -{ - int httpStatusCode = reply->attribute(QNetworkRequest::HttpStatusCodeAttribute).toInt(); - if (httpStatusCode / 100 != 2) { - /* Fall back to full upload */ - qCWarning(lcZsyncPut) << "Failed to retrieve zsync metadata for:" << _item->_file; - _rangesToUpload.clear(); - UploadRangeInfo rangeinfo = { 0, _item->_size }; - _rangesToUpload.append(rangeinfo); - _bytesToUpload = _item->_size; - doStartUploadNext(); - return; - } - - QByteArray zsyncData = reply->readAll(); - - qCInfo(lcZsyncPut) << "Retrieved zsync metadata for:" << _item->_file << "size:" << zsyncData.size(); - - ZsyncSeedRunnable *run = new ZsyncSeedRunnable(zsyncData, propagator()->getFilePath(_item->_file), ZsyncMode::upload); - connect(run, &ZsyncSeedRunnable::finishedSignal, this, &PropagateUploadFileNG::slotZsyncSeedFinished); - connect(run, &ZsyncSeedRunnable::failedSignal, this, &PropagateUploadFileNG::slotZsyncSeedFailed); - - // Starts in a seperate thread - QThreadPool::globalInstance()->start(run); -} void PropagateUploadFileNG::doStartUploadNext() { - if (_zsyncSupported) { - _isZsyncMetadataUploadRunning = true; - - ZsyncGenerateRunnable *run = new ZsyncGenerateRunnable(propagator()->getFilePath(_item->_file)); - connect(run, &ZsyncGenerateRunnable::finishedSignal, this, &PropagateUploadFileNG::slotZsyncGenerationFinished); - connect(run, &ZsyncGenerateRunnable::failedSignal, this, &PropagateUploadFileNG::slotZsyncGenerationFailed); - - // Starts in a seperate thread - QThreadPool::globalInstance()->start(run); - } - const SyncJournalDb::UploadInfo progressInfo = propagator()->_journal->getUploadInfo(_item->_file); if (progressInfo._valid && progressInfo.isChunked() && progressInfo._modtime == _item->_modtime && progressInfo._size == _item->_size) { @@ -422,7 +254,6 @@ void PropagateUploadFileNG::slotDeleteJobFinished() } // If no more Delete jobs are running, we can continue - // (the zsync metadata upload might run in parallel) bool runningDeleteJobs = false; for (auto otherJob : _jobs) { if (qobject_cast<DeleteJob *>(otherJob)) @@ -486,10 +317,6 @@ void PropagateUploadFileNG::slotMkColFinished(QNetworkReply::NetworkError) void PropagateUploadFileNG::doFinalMove() { - // Still not finished metadata upload. - if (_isZsyncMetadataUploadRunning) - return; - // Still not finished all ranges. if (!_rangesToUpload.isEmpty()) return; @@ -514,7 +341,7 @@ void PropagateUploadFileNG::doFinalMove() headers[QByteArrayLiteral("OC-Total-Length")] = QByteArray::number(_bytesToUpload); headers[QByteArrayLiteral("OC-Total-File-Length")] = QByteArray::number(_item->_size); - QUrl source = _zsyncSupported ? Utility::concatUrlPath(chunkUrl(), QStringLiteral("/.file.zsync")) : Utility::concatUrlPath(chunkUrl(), QStringLiteral("/.file")); + const QUrl source = Utility::concatUrlPath(chunkUrl(), QStringLiteral("/.file")); auto job = new MoveJob(propagator()->account(), source, destination, headers, this); _jobs.append(job); @@ -578,62 +405,6 @@ void PropagateUploadFileNG::startNextChunk() propagator()->_activeJobList.append(this); } -void PropagateUploadFileNG::slotZsyncGenerationFinished(const QString &generatedFileName) -{ - qCDebug(lcPropagateUploadNG) - << "Finished generation of:" << generatedFileName - << "size:" << FileSystem::getSize(generatedFileName); - - auto device = std::unique_ptr<UploadDevice>(new UploadDevice( - generatedFileName, 0, FileSystem::getSize(generatedFileName), &propagator()->_bandwidthManager)); - if (!device->open(QIODevice::ReadOnly)) { - qCWarning(lcPropagateUploadNG) << "Could not prepare generated file: " << generatedFileName << device->errorString(); - abortWithError(SyncFileItem::SoftError, device->errorString()); - return; - } - - QMap<QByteArray, QByteArray> headers; - QUrl url = Utility::concatUrlPath(chunkUrl(), ".zsync"); - - _sent += FileSystem::getSize(generatedFileName); - _bytesToUpload += FileSystem::getSize(generatedFileName); - - qCDebug(lcPropagateUploadNG) << "Starting upload of .zsync"; - - // job takes ownership of device via a QScopedPointer. Job deletes itself when finishing - auto devicePtr = device.get(); // for connections later - PUTFileJob *job = new PUTFileJob(propagator()->account(), url, std::move(device), headers, 0, this); - _jobs.append(job); - connect(job, &PUTFileJob::finishedSignal, this, &PropagateUploadFileNG::slotZsyncMetadataUploadFinished); - connect(job, &PUTFileJob::uploadProgress, - this, &PropagateUploadFileNG::slotUploadProgress); - connect(job, &PUTFileJob::uploadProgress, - devicePtr, &UploadDevice::slotJobUploadProgress); - job->start(); - propagator()->_activeJobList.append(this); - - FileSystem::remove(generatedFileName); -} - -void PropagateUploadFileNG::slotZsyncMetadataUploadFinished() -{ - qCDebug(lcPropagateUploadNG) << "Uploading of .zsync complete"; - - PUTFileJob *job = qobject_cast<PUTFileJob *>(sender()); - OC_ASSERT(job); - slotJobDestroyed(job); - - _isZsyncMetadataUploadRunning = false; - doFinalMove(); -} - -void PropagateUploadFileNG::slotZsyncGenerationFailed(const QString &errorString) -{ - qCWarning(lcZsyncPut) << "Failed to generate zsync metadata file:" << errorString; - - abortWithError(SyncFileItem::SoftError, tr("Failed to generate zsync file.")); -} - void PropagateUploadFileNG::slotPutFinished() { PUTFileJob *job = qobject_cast<PUTFileJob *>(sender()); diff --git a/src/libsync/syncoptions.h b/src/libsync/syncoptions.h index c4fc50bf5..47dd21e34 100644 --- a/src/libsync/syncoptions.h +++ b/src/libsync/syncoptions.h @@ -67,12 +67,6 @@ struct OWNCLOUDSYNC_EXPORT SyncOptions /** The maximum number of active jobs in parallel */ int _parallelNetworkJobs = 6; - /** Whether delta-synchronization is enabled */ - bool _deltaSyncEnabled = false; - - /** What the minimum file size (in Bytes) is for delta-synchronization */ - qint64 _deltaSyncMinFileSize = 0; - /** Reads settings from env vars where available. * * Currently reads _initialChunkSize, _minChunkSize, _maxChunkSize, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 993a23f08..f0d2bfe7d 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -22,7 +22,6 @@ owncloud_add_test(SyncConflict "syncenginetestutils.h") owncloud_add_test(SyncFileStatusTracker "syncenginetestutils.h") owncloud_add_test(Download "syncenginetestutils.h") owncloud_add_test(ChunkingNg "syncenginetestutils.h") -owncloud_add_test(Zsync "syncenginetestutils.h") owncloud_add_test(AsyncOp "syncenginetestutils.h") owncloud_add_test(UploadReset "syncenginetestutils.h") owncloud_add_test(AllFilesDeleted "syncenginetestutils.h") diff --git a/test/syncenginetestutils.h b/test/syncenginetestutils.h index 858660c95..724dbf830 100644 --- a/test/syncenginetestutils.h +++ b/test/syncenginetestutils.h @@ -406,7 +406,6 @@ public: : fileInfo.isShared ? QStringLiteral("SRDNVCKW") : QStringLiteral("RDNVCKW")); xml.writeTextElement(ocUri, QStringLiteral("id"), fileInfo.fileId); xml.writeTextElement(ocUri, QStringLiteral("checksums"), fileInfo.checksums); - xml.writeTextElement(ocUri, QStringLiteral("zsync"), QStringLiteral("true")); buffer.write(fileInfo.extraDavProperties); xml.writeEndElement(); // prop xml.writeTextElement(davUri, QStringLiteral("status"), "HTTP/1.1 200 OK"); @@ -764,15 +763,10 @@ public: static FileInfo *perform(FileInfo &uploadsFileInfo, FileInfo &remoteRootFileInfo, const QNetworkRequest &request) { QString source = getFilePathFromUrl(request.url()); - bool zsync = false; Q_ASSERT(!source.isEmpty()); - Q_ASSERT(source.endsWith("/.file") || source.endsWith("/.file.zsync")); - if (source.endsWith("/.file")) - source = source.left(source.length() - qstrlen("/.file")); - if (source.endsWith("/.file.zsync")) { - source = source.left(source.length() - qstrlen("/.file.zsync")); - zsync = true; - } + Q_ASSERT(source.endsWith("/.file")); + source = source.left(source.length() - qstrlen("/.file")); + auto sourceFolder = uploadsFileInfo.find(source); Q_ASSERT(sourceFolder); Q_ASSERT(sourceFolder->isDir); @@ -784,34 +778,21 @@ public: QString fileName = getFilePathFromUrl(QUrl::fromEncoded(request.rawHeader("Destination"))); Q_ASSERT(!fileName.isEmpty()); - // Ignore .zsync metadata - if (sourceFolder->children.contains(".zsync")) - sourceFolder->children.remove(".zsync"); - // Compute the size and content from the chunks if possible for (auto chunkName : sourceFolder->children.keys()) { auto &x = sourceFolder->children[chunkName]; - if (!zsync && chunkName.toLongLong() != prev) + if (chunkName.toLongLong() != prev) break; Q_ASSERT(!x.isDir); Q_ASSERT(x.size > 0); // There should not be empty chunks size += x.size; - Q_ASSERT(!payload || payload == x.contentChar || !"For zsync all chunks must start with the same character"); + Q_ASSERT(!payload || payload == x.contentChar); payload = x.contentChar; ++count; prev = chunkName.toLongLong() + x.size; } Q_ASSERT(sourceFolder->children.count() == count); // There should not be holes or extra files - // For zsync, get the size from the header, and allow no-chunk uploads (shrinking files) - if (zsync) { - size = request.rawHeader("OC-Total-File-Length").toLongLong(); - if (count == 0) { - if (auto info = remoteRootFileInfo.find(fileName)) - payload = info->contentChar; - } - } - // NOTE: This does not actually assemble the file data from the chunks! FileInfo *fileInfo = remoteRootFileInfo.find(fileName); if (fileInfo) { @@ -864,99 +845,6 @@ public: qint64 readData(char *, qint64) override { return 0; } }; -class FakeChunkZsyncMoveReply : public QNetworkReply -{ - Q_OBJECT - FileInfo *fileInfo; - -public: - FakeChunkZsyncMoveReply(FileInfo &uploadsFileInfo, FileInfo &remoteRootFileInfo, - QNetworkAccessManager::Operation op, const QNetworkRequest &request, - quint64 delayMs, QVector<quint64> &mods, QObject *parent) - : QNetworkReply{ parent } - { - setRequest(request); - setUrl(request.url()); - setOperation(op); - open(QIODevice::ReadOnly); - - Q_ASSERT(!mods.isEmpty()); - - QString source = getFilePathFromUrl(request.url()); - Q_ASSERT(!source.isEmpty()); - Q_ASSERT(source.endsWith("/.file.zsync")); - source = source.left(source.length() - qstrlen("/.file.zsync")); - auto sourceFolder = uploadsFileInfo.find(source); - Q_ASSERT(sourceFolder); - Q_ASSERT(sourceFolder->isDir); - int count = 0; - - // Ignore .zsync metadata - if (sourceFolder->children.contains(".zsync")) - sourceFolder->children.remove(".zsync"); - - for (auto chunkName : sourceFolder->children.keys()) { - auto &x = sourceFolder->children[chunkName]; - Q_ASSERT(!x.isDir); - Q_ASSERT(x.size > 0); // There should not be empty chunks - quint64 start = quint64(chunkName.toLongLong()); - auto it = mods.begin(); - while (it != mods.end()) { - if (*it >= start && *it < start + x.size) { - ++count; - mods.erase(it); - } else - ++it; - } - } - - Q_ASSERT(count > 0); // There should be at least one chunk - Q_ASSERT(mods.isEmpty()); // All files should match a modification - - QString fileName = getFilePathFromUrl(QUrl::fromEncoded(request.rawHeader("Destination"))); - Q_ASSERT(!fileName.isEmpty()); - - fileInfo = remoteRootFileInfo.find(fileName); - Q_ASSERT(fileInfo); - if (!fileInfo) { - abort(); - return; - } - - QVERIFY(request.hasRawHeader("If")); // The client should put this header - if (request.rawHeader("If") != QByteArray("<" + request.rawHeader("Destination") + "> ([\"" + fileInfo->etag.toLatin1() + "\"])")) { - QMetaObject::invokeMethod(this, "respondPreconditionFailed", Qt::QueuedConnection); - return; - } - - fileInfo->lastModified = OCC::Utility::qDateTimeFromTime_t(request.rawHeader("X-OC-Mtime").toLongLong()); - remoteRootFileInfo.find(fileName, /*invalidate_etags=*/true); - - QTimer::singleShot(delayMs, this, &FakeChunkZsyncMoveReply::respond); - } - - Q_INVOKABLE void respond() - { - setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 201); - setRawHeader("OC-ETag", fileInfo->etag.toLatin1()); - setRawHeader("ETag", fileInfo->etag.toLatin1()); - setRawHeader("OC-FileId", fileInfo->fileId); - emit metaDataChanged(); - emit finished(); - } - - Q_INVOKABLE void respondPreconditionFailed() - { - setAttribute(QNetworkRequest::HttpStatusCodeAttribute, 412); - setError(InternalServerError, "Precondition Failed"); - emit metaDataChanged(); - emit finished(); - } - - void abort() override {} - qint64 readData(char *, qint64) override { return 0; } -}; - class FakePayloadReply : public QNetworkReply { Q_OBJECT diff --git a/test/testzsync.cpp b/test/testzsync.cpp deleted file mode 100644 index 4aa2dbaed..000000000 --- a/test/testzsync.cpp +++ /dev/null @@ -1,371 +0,0 @@ -/* - * This software is in the public domain, furnished "as is", without technical - * support, and with no warranty, express or implied, as to its usefulness for - * any purpose. - * - */ - -#include <QtTest> -#include "syncenginetestutils.h" -#include <syncengine.h> -#include <propagatecommonzsync.h> - -using namespace OCC; - -QStringList findConflicts(const FileInfo &dir) -{ - QStringList conflicts; - for (const auto &item : dir.children) { - if (item.name.contains("conflict")) { - conflicts.append(item.path()); - } - } - return conflicts; -} - -static quint64 blockstart_from_offset(quint64 offset) -{ - return offset & ~quint64(ZSYNC_BLOCKSIZE - 1); -} - -class TestZsync : public QObject -{ - Q_OBJECT - -private slots: - - void testFileDownloadSimple() - { - FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; - fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" }, { "zsync", "1.0" } } } }); - - SyncOptions opt; - opt._deltaSyncEnabled = true; - opt._deltaSyncMinFileSize = 0; - // Reduce the chunk size, allowing us to work with smaller data - int chunkSize = 2 * 1000 * 1001; - opt._minChunkSize = opt._maxChunkSize = opt._initialChunkSize = chunkSize; - fakeFolder.syncEngine().setSyncOptions(opt); - - const int size = 100 * 1000 * 1000; - QMap<QString, QByteArray> metadata; - - // Preparation, create files and upload them, capturing metadata - // (because letting the client compute the metadata is easiest) - - fakeFolder.localModifier().insert("A/a0", size); - fakeFolder.localModifier().appendByte("A/a0", 'X'); - qsrand(QDateTime::currentDateTime().toTime_t()); - const int nModifications = 10; - for (int i = 0; i < nModifications; i++) { - quint64 offset = qrand() % size; - fakeFolder.localModifier().modifyByte("A/a0", offset, 'Y'); - } - - // Keep hold of original file contents for a0 - QFile f(fakeFolder.localPath() + "/A/a0"); - f.open(QIODevice::ReadOnly); - QByteArray data = f.readAll(); - f.close(); - - fakeFolder.localModifier().insert("A/threeBlocks", 3 * ZSYNC_BLOCKSIZE); - fakeFolder.localModifier().insert("A/threeBlocksPlusOne", 3 * ZSYNC_BLOCKSIZE + 1); - fakeFolder.localModifier().insert("A/threeBlocksMinusOne", 3 * ZSYNC_BLOCKSIZE - 1); - - // Capture the zsync metadata during upload for later use - // This fills metadata by filename - auto transactionId = [] (const QUrl &url) { - auto components = url.path().split("/"); - return components[components.size() - 2]; - }; - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *data) -> QNetworkReply * { - if (op == QNetworkAccessManager::PutOperation && request.url().toString().endsWith(".zsync")) { - metadata[transactionId(request.url())] = data->readAll(); - return new FakePutReply{ fakeFolder.uploadState(), op, request, metadata[transactionId(request.url())], this }; - } - if (request.attribute(QNetworkRequest::CustomVerbAttribute) == "MOVE") { - metadata[request.rawHeader("Destination").split('/').last()] = metadata[transactionId(request.url())]; - } - return nullptr; - }); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(metadata["a0"].startsWith("zsync: ")); - QVERIFY(metadata["threeBlocks"].startsWith("zsync: ")); - QVERIFY(metadata["threeBlocksPlusOne"].startsWith("zsync: ")); - QVERIFY(metadata["threeBlocksMinusOne"].startsWith("zsync: ")); - - - // Test 1: create a conflict by changing the local file and touching the remote - // and verify zsync saves bandwidth on download - - // The new local file is like the original, - // but without the modifications and the appendByte - fakeFolder.localModifier().remove("A/a0"); - fakeFolder.localModifier().insert("A/a0", size); - auto currentMtime = QDateTime::currentDateTimeUtc(); - fakeFolder.remoteModifier().setModTime("A/a0", currentMtime); - quint64 transferedData = 0; - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * { - QUrlQuery query(request.url()); - if (op == QNetworkAccessManager::GetOperation) { - if (query.hasQueryItem("zsync")) { - return new FakeGetWithDataReply{ fakeFolder.remoteModifier(), metadata["a0"], op, request, this }; - } - - auto reply = new FakeGetWithDataReply{ fakeFolder.remoteModifier(), data, op, request, this }; - transferedData += reply->payload.size(); - return reply; - } - - return nullptr; - }); - QVERIFY(fakeFolder.syncOnce()); - - // We didn't transfer the whole file - // (plus one because of the new trailing byte) - QVERIFY(transferedData <= (nModifications + 1) * ZSYNC_BLOCKSIZE); - - // Verify that the newly propagated file was assembled to have the expected data - f.open(QIODevice::ReadOnly); - QVERIFY(data == f.readAll()); - f.close(); - - // Check the conflict creation - auto conflicts = findConflicts(fakeFolder.currentLocalState().children["A"]); - QCOMPARE(conflicts.size(), 1); - for (auto c : conflicts) { - fakeFolder.localModifier().remove(c); - } - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - - - // Test 2: The remote file has grown, not on a zsync block boundary - - // Setup the local state - fakeFolder.remoteModifier().insert("A/growthSameBlock", 3 * ZSYNC_BLOCKSIZE - 1); - fakeFolder.remoteModifier().insert("A/growthNewBlock", 3 * ZSYNC_BLOCKSIZE); - fakeFolder.remoteModifier().insert("A/shrinkSameBlock", 3 * ZSYNC_BLOCKSIZE); - fakeFolder.remoteModifier().insert("A/shrinkFewerBlock", 3 * ZSYNC_BLOCKSIZE + 1); - QVERIFY(fakeFolder.syncOnce()); - - // Grow/shrink files on the remote side - fakeFolder.remoteModifier().remove("A/growthSameBlock"); - fakeFolder.remoteModifier().remove("A/growthNewBlock"); - fakeFolder.remoteModifier().remove("A/shrinkSameBlock"); - fakeFolder.remoteModifier().remove("A/shrinkFewerBlock"); - fakeFolder.remoteModifier().insert("A/growthSameBlock", 3 * ZSYNC_BLOCKSIZE); - fakeFolder.remoteModifier().insert("A/growthNewBlock", 3 * ZSYNC_BLOCKSIZE + 1); - fakeFolder.remoteModifier().insert("A/shrinkSameBlock", 3 * ZSYNC_BLOCKSIZE - 1); - fakeFolder.remoteModifier().insert("A/shrinkFewerBlock", 3 * ZSYNC_BLOCKSIZE); - - // inject appropriate zsync metadata and actual data replies - QMap<QString, QByteArray> downloads; - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * { - QUrlQuery query(request.url()); - if (op == QNetworkAccessManager::GetOperation) { - auto filename = request.url().path().split("/").last(); - if (query.hasQueryItem("zsync")) { - QByteArray meta; - if (filename == "growthSameBlock") { - meta = metadata["threeBlocks"]; - } else if (filename == "growthNewBlock") { - meta = metadata["threeBlocksPlusOne"]; - } else if (filename == "shrinkSameBlock") { - meta = metadata["threeBlocksMinusOne"]; - } else if (filename == "shrinkFewerBlock") { - meta = metadata["threeBlocks"]; - } else { - Q_ASSERT(false); - } - return new FakeGetWithDataReply{ fakeFolder.remoteModifier(), meta, op, request, this }; - } else { - downloads[filename] = request.rawHeader("Range"); - QByteArray fakeData(3 * ZSYNC_BLOCKSIZE + 1, fakeFolder.remoteModifier().find("A/growthNewBlock")->contentChar); - return new FakeGetWithDataReply{ fakeFolder.remoteModifier(), fakeData, op, request, this }; - } - } - return nullptr; - }); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - - // growthSameBlock: no download needed, first block can become third block - QVERIFY(!downloads.contains("growthSameBlock")); - // growthNewBlock: downloading one byte - QCOMPARE(downloads["growthNewBlock"].data(), "bytes=3145728-3145728"); - // shrinkSameBlock: downloading N-1 bytes - QCOMPARE(downloads["shrinkSameBlock"].data(), "bytes=2097152-3145726"); - // shrinkFewerBlock: no download needed - QVERIFY(!downloads.contains("shrinkFewerBlock")); - } - - void testFileUploadSimple() - { - FakeFolder fakeFolder{ FileInfo::A12_B12_C12_S12() }; - fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" }, { "zsync", "1.0" } } } }); - - SyncOptions opt; - opt._deltaSyncEnabled = true; - opt._deltaSyncMinFileSize = 0; - fakeFolder.syncEngine().setSyncOptions(opt); - - const int size = 100 * 1000 * 1000; - QByteArray metadata; - - // Test 1: NEW file upload with zsync metadata - fakeFolder.localModifier().insert("A/a0", size); - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *data) -> QNetworkReply * { - if (op == QNetworkAccessManager::PutOperation && request.url().toString().endsWith(".zsync")) { - metadata = data->readAll(); - return new FakePutReply{ fakeFolder.uploadState(), op, request, metadata, this }; - } - - return nullptr; - }); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(metadata.startsWith("zsync: ")); - - // Test 2: Modify local contents and ensure that modified chunks are sent - QVector<quint64> mods; - qsrand(QDateTime::currentDateTime().toTime_t()); - fakeFolder.localModifier().appendByte("A/a0", 'X'); - mods.append(blockstart_from_offset(size + 1)); - for (int i = 0; i < 10; i++) { - quint64 offset = qrand() % size; - fakeFolder.localModifier().modifyByte("A/a0", offset, 'Y'); - mods.append(blockstart_from_offset(offset)); - } - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *) -> QNetworkReply * { - QUrlQuery query(request.url()); - if (op == QNetworkAccessManager::GetOperation && query.hasQueryItem("zsync")) { - return new FakeGetWithDataReply{ fakeFolder.remoteModifier(), metadata, op, request, this }; - } - - if (request.attribute(QNetworkRequest::CustomVerbAttribute) == QLatin1String("MOVE")) { - return new FakeChunkZsyncMoveReply{ fakeFolder.uploadState(), fakeFolder.remoteModifier(), op, request, 0, mods, this }; - } - - return nullptr; - }); - QVERIFY(fakeFolder.syncOnce()); - fakeFolder.remoteModifier().appendByte("A/a0", 'X'); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - } - - void testFileUploadGrowShrink() - { - FakeFolder fakeFolder{ FileInfo() }; - fakeFolder.syncEngine().account()->setCapabilities({ { "dav", QVariantMap{ { "chunking", "1.0" }, { "zsync", "1.0" } } } }); - - SyncOptions opt; - opt._deltaSyncEnabled = true; - opt._deltaSyncMinFileSize = 0; - int chunkSize = 2 * 1000 * 1000; - opt._minChunkSize = opt._maxChunkSize = opt._initialChunkSize = chunkSize; // don't dynamically size chunks - fakeFolder.syncEngine().setSyncOptions(opt); - - const int zsyncBlockSize = 1024 * 1024; - - QByteArray metadata; - QList<QPair<int, int>> putChunks; - fakeFolder.setServerOverride([&](QNetworkAccessManager::Operation op, const QNetworkRequest &request, QIODevice *data) -> QNetworkReply * { - // Handle .zsync file storage and retrieval - note that preparing a PUT already updates the - // zsync file data before the upload is finalized with a MOVE here. - QUrlQuery query(request.url()); - if (request.url().toString().endsWith(".zsync") && op == QNetworkAccessManager::PutOperation) { - metadata = data->readAll(); - return new FakePutReply{ fakeFolder.uploadState(), op, request, metadata, this }; - } - if (op == QNetworkAccessManager::GetOperation && query.hasQueryItem("zsync")) { - return new FakeGetWithDataReply{ fakeFolder.remoteModifier(), metadata, op, request, this }; - } - // Grab chunk offset and size - if (op == QNetworkAccessManager::PutOperation) { - auto payload = data->readAll(); - putChunks.append({ request.rawHeader("OC-Chunk-Offset").toInt(), payload.size() }); - return new FakePutReply{ fakeFolder.uploadState(), op, request, payload, this }; - } - return nullptr; - }); - - // Note: The remote side doesn't actually store the file contents - - // Test 1: NEW file upload with zsync metadata - auto totalSize = 2 * zsyncBlockSize + 5; - fakeFolder.localModifier().insert("a0", totalSize); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 2); - QVERIFY(putChunks[0] == qMakePair(0, 2000000)); - QVERIFY(putChunks[1] == qMakePair(2000000, totalSize - 2000000)); - - // Test 2: Appending data to the file - putChunks.clear(); - totalSize += 1; - fakeFolder.localModifier().appendByte("a0", 'Q'); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 1); - QVERIFY(putChunks[0] == qMakePair(2 * zsyncBlockSize, 6)); - - // Test 3: reduce the file size - putChunks.clear(); - totalSize -= 10; - fakeFolder.localModifier().remove("a0"); - fakeFolder.localModifier().insert("a0", totalSize); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 1); - QVERIFY(putChunks[0] == qMakePair(zsyncBlockSize, totalSize - zsyncBlockSize)); // technically unnecessary - - // Test 4: add a large amount, such that the zsync block gets chunked - putChunks.clear(); - totalSize += int(1.5 * chunkSize); - fakeFolder.localModifier().remove("a0"); - fakeFolder.localModifier().insert("a0", totalSize); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 3); - // The first block stays, the rest is re-uploaded, total upload size is roughly 1MiB + 1.5 * 2MB = slightly more than 4MB - QVERIFY(putChunks[0] == qMakePair(zsyncBlockSize, chunkSize)); - QVERIFY(putChunks[1] == qMakePair(zsyncBlockSize + chunkSize, chunkSize)); - QVERIFY(putChunks[2] == qMakePair(zsyncBlockSize + 2 * chunkSize, totalSize - (zsyncBlockSize + 2 * chunkSize))); - - // Test 5: append and change an early block at the same time - putChunks.clear(); - totalSize += 1; - fakeFolder.localModifier().appendByte("a0", 'Q'); - fakeFolder.localModifier().modifyByte("a0", 5, 'Q'); - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 2); - QVERIFY(putChunks[0] == qMakePair(0, zsyncBlockSize)); - QVERIFY(putChunks[1] == qMakePair(4 * zsyncBlockSize, totalSize - 4 * zsyncBlockSize)); - - // Test 6: Shrink to an aligned size - putChunks.clear(); - totalSize = 2 * zsyncBlockSize; - fakeFolder.localModifier().remove("a0"); - fakeFolder.localModifier().insert("a0", totalSize); - fakeFolder.localModifier().modifyByte("a0", 5, 'Q'); // same data as before - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 0); - - // Test 7: Grow to an aligned size - putChunks.clear(); - totalSize = 3 * zsyncBlockSize; - fakeFolder.localModifier().remove("a0"); - fakeFolder.localModifier().insert("a0", totalSize); - fakeFolder.localModifier().modifyByte("a0", 5, 'Q'); // same data as before - QVERIFY(fakeFolder.syncOnce()); - QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); - QVERIFY(putChunks.size() == 1); - QVERIFY(putChunks[0] == qMakePair(2 * zsyncBlockSize, zsyncBlockSize)); - } -}; - -QTEST_GUILESS_MAIN(TestZsync) -#include "testzsync.moc" |