diff options
34 files changed, 377 insertions, 1396 deletions
diff --git a/src/common/syncjournaldb.cpp b/src/common/syncjournaldb.cpp index 2146e6945..d2b6fb2a7 100644 --- a/src/common/syncjournaldb.cpp +++ b/src/common/syncjournaldb.cpp @@ -36,6 +36,26 @@ namespace OCC { Q_LOGGING_CATEGORY(lcDb, "sync.database", QtInfoMsg) +#define GET_FILE_RECORD_QUERY \ + "SELECT path, inode, modtime, type, md5, fileid, remotePerm, filesize," \ + " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" \ + " FROM metadata" \ + " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" + +static void fillFileRecordFromGetQuery(SyncJournalFileRecord &rec, SqlQuery &query) +{ + rec._path = query.baValue(0); + rec._inode = query.intValue(1); + rec._modtime = query.int64Value(2); + rec._type = query.intValue(3); + rec._etag = query.baValue(4); + rec._fileId = query.baValue(5); + rec._remotePerm = RemotePermissions(query.baValue(6).constData()); + rec._fileSize = query.int64Value(7); + rec._serverHasIgnoredFiles = (query.intValue(8) > 0); + rec._checksumHeader = query.baValue(9); +} + static QString defaultJournalMode(const QString &dbPath) { #ifdef Q_OS_WIN @@ -500,14 +520,32 @@ bool SyncJournalDb::checkConnect() _getFileRecordQuery.reset(new SqlQuery(_db)); if (_getFileRecordQuery->prepare( - "SELECT path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize," - " ignoredChildrenRemote, contentchecksumtype.name || ':' || contentChecksum" - " FROM metadata" - " LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" + GET_FILE_RECORD_QUERY " WHERE phash=?1")) { return sqlFail("prepare _getFileRecordQuery", *_getFileRecordQuery); } + _getFileRecordQueryByInode.reset(new SqlQuery(_db)); + if (_getFileRecordQueryByInode->prepare( + GET_FILE_RECORD_QUERY + " WHERE inode=?1")) { + return sqlFail("prepare _getFileRecordQueryByInode", *_getFileRecordQueryByInode); + } + + _getFileRecordQueryByFileId.reset(new SqlQuery(_db)); + if (_getFileRecordQueryByFileId->prepare( + GET_FILE_RECORD_QUERY + " WHERE fileid=?1")) { + return sqlFail("prepare _getFileRecordQueryByFileId", *_getFileRecordQueryByFileId); + } + + _getFilesBelowPathQuery.reset(new SqlQuery(_db)); + if (_getFilesBelowPathQuery->prepare( + GET_FILE_RECORD_QUERY + " WHERE path > (?1||'/') AND path < (?1||'0') ORDER BY path||'/' ASC")) { + return sqlFail("prepare _getFilesBelowPathQuery", *_getFilesBelowPathQuery); + } + _setFileRecordQuery.reset(new SqlQuery(_db)); if (_setFileRecordQuery->prepare("INSERT OR REPLACE INTO metadata " "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5, fileid, remotePerm, filesize, ignoredChildrenRemote, contentChecksum, contentChecksumTypeId) " @@ -651,6 +689,9 @@ void SyncJournalDb::close() commitTransaction(); _getFileRecordQuery.reset(0); + _getFileRecordQueryByInode.reset(0); + _getFileRecordQueryByFileId.reset(0); + _getFilesBelowPathQuery.reset(0); _setFileRecordQuery.reset(0); _setFileRecordChecksumQuery.reset(0); _setFileRecordLocalMetadataQuery.reset(0); @@ -860,18 +901,17 @@ QStringList SyncJournalDb::tableColumns(const QString &table) return columns; } -qint64 SyncJournalDb::getPHash(const QString &file) +qint64 SyncJournalDb::getPHash(const QByteArray &file) { - QByteArray utf8File = file.toUtf8(); int64_t h; if (file.isEmpty()) { return -1; } - int len = utf8File.length(); + int len = file.length(); - h = c_jhash64((uint8_t *)utf8File.data(), len, 0); + h = c_jhash64((uint8_t *)file.data(), len, 0); return h; } @@ -882,8 +922,8 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) if (!_avoidReadFromDbOnNextSyncFilter.isEmpty()) { // If we are a directory that should not be read from db next time, don't write the etag - QString prefix = record._path + "/"; - foreach (const QString &it, _avoidReadFromDbOnNextSyncFilter) { + QByteArray prefix = record._path + "/"; + foreach (const QByteArray &it, _avoidReadFromDbOnNextSyncFilter) { if (it.startsWith(prefix)) { qCInfo(lcDb) << "Filtered writing the etag of" << prefix << "because it is a prefix of" << it; record._etag = "_invalid_"; @@ -899,13 +939,12 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) qlonglong phash = getPHash(record._path); if (checkConnect()) { - QByteArray arr = record._path.toUtf8(); - int plen = arr.length(); + int plen = record._path.length(); - QString etag(record._etag); + QByteArray etag(record._etag); if (etag.isEmpty()) etag = ""; - QString fileId(record._fileId); + QByteArray fileId(record._fileId); if (fileId.isEmpty()) fileId = ""; QByteArray remotePerm = record._remotePerm.toString(); @@ -920,8 +959,8 @@ bool SyncJournalDb::setFileRecord(const SyncJournalFileRecord &_record) _setFileRecordQuery->bindValue(5, 0); // uid Not used _setFileRecordQuery->bindValue(6, 0); // gid Not used _setFileRecordQuery->bindValue(7, 0); // mode Not used - _setFileRecordQuery->bindValue(8, QString::number(Utility::qDateTimeToTime_t(record._modtime))); - _setFileRecordQuery->bindValue(9, QString::number(record._type)); + _setFileRecordQuery->bindValue(8, record._modtime); + _setFileRecordQuery->bindValue(9, record._type); _setFileRecordQuery->bindValue(10, etag); _setFileRecordQuery->bindValue(11, fileId); _setFileRecordQuery->bindValue(12, remotePerm); @@ -949,7 +988,7 @@ bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively) // if (!recursively) { // always delete the actual file. - qlonglong phash = getPHash(filename); + qlonglong phash = getPHash(filename.toUtf8()); _deleteFileRecordPhash->reset_and_clear_bindings(); _deleteFileRecordPhash->bindValue(1, phash); @@ -972,7 +1011,7 @@ bool SyncJournalDb::deleteFileRecord(const QString &filename, bool recursively) } -bool SyncJournalDb::getFileRecord(const QString &filename, SyncJournalFileRecord *rec) +bool SyncJournalDb::getFileRecord(const QByteArray &filename, SyncJournalFileRecord *rec) { QMutexLocker locker(&_mutex); @@ -995,19 +1034,7 @@ bool SyncJournalDb::getFileRecord(const QString &filename, SyncJournalFileRecord } if (_getFileRecordQuery->next()) { - rec->_path = _getFileRecordQuery->stringValue(0); - rec->_inode = _getFileRecordQuery->intValue(1); - //rec->_uid = _getFileRecordQuery->value(2).toInt(&ok); Not Used - //rec->_gid = _getFileRecordQuery->value(3).toInt(&ok); Not Used - //rec->_mode = _getFileRecordQuery->intValue(4); - rec->_modtime = Utility::qDateTimeFromTime_t(_getFileRecordQuery->int64Value(5)); - rec->_type = _getFileRecordQuery->intValue(6); - rec->_etag = _getFileRecordQuery->baValue(7); - rec->_fileId = _getFileRecordQuery->baValue(8); - rec->_remotePerm = RemotePermissions(_getFileRecordQuery->baValue(9).constData()); - rec->_fileSize = _getFileRecordQuery->int64Value(10); - rec->_serverHasIgnoredFiles = (_getFileRecordQuery->intValue(11) > 0); - rec->_checksumHeader = _getFileRecordQuery->baValue(12); + fillFileRecordFromGetQuery(*rec, *_getFileRecordQuery); } else { int errId = _getFileRecordQuery->errorId(); if (errId != SQLITE_DONE) { // only do this if the problem is different from SQLITE_DONE @@ -1022,6 +1049,87 @@ bool SyncJournalDb::getFileRecord(const QString &filename, SyncJournalFileRecord return true; } +bool SyncJournalDb::getFileRecordByInode(quint64 inode, SyncJournalFileRecord *rec) +{ + QMutexLocker locker(&_mutex); + + // Reset the output var in case the caller is reusing it. + Q_ASSERT(rec); + rec->_path.clear(); + Q_ASSERT(!rec->isValid()); + + if (!inode) + return true; // no error, yet nothing found (rec->isValid() == false) + + if (!checkConnect()) + return false; + + _getFileRecordQueryByInode->reset_and_clear_bindings(); + _getFileRecordQueryByInode->bindValue(1, inode); + + if (!_getFileRecordQueryByInode->exec()) { + return false; + } + + if (_getFileRecordQueryByInode->next()) { + fillFileRecordFromGetQuery(*rec, *_getFileRecordQueryByInode); + } + + return true; +} + +bool SyncJournalDb::getFileRecordByFileId(const QByteArray &fileId, SyncJournalFileRecord *rec) +{ + QMutexLocker locker(&_mutex); + + // Reset the output var in case the caller is reusing it. + Q_ASSERT(rec); + rec->_path.clear(); + Q_ASSERT(!rec->isValid()); + + if (fileId.isEmpty()) + return true; // no error, yet nothing found (rec->isValid() == false) + + if (!checkConnect()) + return false; + + _getFileRecordQueryByFileId->reset_and_clear_bindings(); + _getFileRecordQueryByFileId->bindValue(1, fileId); + + if (!_getFileRecordQueryByFileId->exec()) { + return false; + } + + if (_getFileRecordQueryByFileId->next()) { + fillFileRecordFromGetQuery(*rec, *_getFileRecordQueryByFileId); + } + + return true; +} + +bool SyncJournalDb::getFilesBelowPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback) +{ + QMutexLocker locker(&_mutex); + + if (!checkConnect()) + return false; + + _getFilesBelowPathQuery->reset_and_clear_bindings(); + _getFilesBelowPathQuery->bindValue(1, path); + + if (!_getFilesBelowPathQuery->exec()) { + return false; + } + + while (_getFilesBelowPathQuery->next()) { + SyncJournalFileRecord rec; + fillFileRecordFromGetQuery(rec, *_getFilesBelowPathQuery); + rowCallback(rec); + } + + return true; +} + bool SyncJournalDb::postSyncCleanup(const QSet<QString> &filepathsToKeep, const QSet<QString> &prefixesToKeep) { @@ -1038,10 +1146,10 @@ bool SyncJournalDb::postSyncCleanup(const QSet<QString> &filepathsToKeep, return false; } - QStringList superfluousItems; + QByteArrayList superfluousItems; while (query.next()) { - const QString file = query.stringValue(1); + const QString file = query.baValue(1); bool keep = filepathsToKeep.contains(file); if (!keep) { foreach (const QString &prefix, prefixesToKeep) { @@ -1052,12 +1160,12 @@ bool SyncJournalDb::postSyncCleanup(const QSet<QString> &filepathsToKeep, } } if (!keep) { - superfluousItems.append(query.stringValue(0)); + superfluousItems.append(query.baValue(0)); } } if (superfluousItems.count()) { - QString sql = "DELETE FROM metadata WHERE phash in (" + superfluousItems.join(",") + ")"; + QByteArray sql = "DELETE FROM metadata WHERE phash in (" + superfluousItems.join(",") + ")"; qCInfo(lcDb) << "Sync Journal cleanup for" << superfluousItems; SqlQuery delQuery(_db); delQuery.prepare(sql); @@ -1103,7 +1211,7 @@ bool SyncJournalDb::updateFileRecordChecksum(const QString &filename, qCInfo(lcDb) << "Updating file checksum" << filename << contentChecksum << contentChecksumType; - qlonglong phash = getPHash(filename); + qlonglong phash = getPHash(filename.toUtf8()); if (!checkConnect()) { qCWarning(lcDb) << "Failed to connect database."; return false; @@ -1132,7 +1240,7 @@ bool SyncJournalDb::updateLocalMetadata(const QString &filename, qCInfo(lcDb) << "Updating local metadata for:" << filename << modtime << size << inode; - qlonglong phash = getPHash(filename); + qlonglong phash = getPHash(filename.toUtf8()); if (!checkConnect()) { qCWarning(lcDb) << "Failed to connect database."; return false; @@ -1328,7 +1436,7 @@ SyncJournalDb::UploadInfo SyncJournalDb::getUploadInfo(const QString &file) res._transferid = _getUploadInfoQuery->intValue(1); res._errorCount = _getUploadInfoQuery->intValue(2); res._size = _getUploadInfoQuery->int64Value(3); - res._modtime = Utility::qDateTimeFromTime_t(_getUploadInfoQuery->int64Value(4)); + res._modtime = _getUploadInfoQuery->int64Value(4); res._valid = ok; } } @@ -1350,7 +1458,7 @@ void SyncJournalDb::setUploadInfo(const QString &file, const SyncJournalDb::Uplo _setUploadInfoQuery->bindValue(3, i._transferid); _setUploadInfoQuery->bindValue(4, i._errorCount); _setUploadInfoQuery->bindValue(5, i._size); - _setUploadInfoQuery->bindValue(6, Utility::qDateTimeToTime_t(i._modtime)); + _setUploadInfoQuery->bindValue(6, i._modtime); if (!_setUploadInfoQuery->exec()) { return; @@ -1539,11 +1647,11 @@ void SyncJournalDb::setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord _setErrorBlacklistQuery->reset_and_clear_bindings(); _setErrorBlacklistQuery->bindValue(1, item._file); _setErrorBlacklistQuery->bindValue(2, item._lastTryEtag); - _setErrorBlacklistQuery->bindValue(3, QString::number(item._lastTryModtime)); + _setErrorBlacklistQuery->bindValue(3, item._lastTryModtime); _setErrorBlacklistQuery->bindValue(4, item._retryCount); _setErrorBlacklistQuery->bindValue(5, item._errorString); - _setErrorBlacklistQuery->bindValue(6, QString::number(item._lastTryTime)); - _setErrorBlacklistQuery->bindValue(7, QString::number(item._ignoreDuration)); + _setErrorBlacklistQuery->bindValue(6, item._lastTryTime); + _setErrorBlacklistQuery->bindValue(7, item._ignoreDuration); _setErrorBlacklistQuery->bindValue(8, item._renameTarget); _setErrorBlacklistQuery->bindValue(9, item._errorCategory); _setErrorBlacklistQuery->exec(); @@ -1591,7 +1699,7 @@ void SyncJournalDb::setPollInfo(const SyncJournalDb::PollInfo &info) } else { SqlQuery query("INSERT OR REPLACE INTO poll (path, modtime, pollpath) VALUES( ? , ? , ? )", _db); query.bindValue(1, info._file); - query.bindValue(2, QString::number(info._modtime)); + query.bindValue(2, info._modtime); query.bindValue(3, info._url); query.exec(); } @@ -1651,7 +1759,7 @@ void SyncJournalDb::setSelectiveSyncList(SyncJournalDb::SelectiveSyncListType ty } } -void SyncJournalDb::avoidRenamesOnNextSync(const QString &path) +void SyncJournalDb::avoidRenamesOnNextSync(const QByteArray &path) { QMutexLocker locker(&_mutex); @@ -1671,7 +1779,7 @@ void SyncJournalDb::avoidRenamesOnNextSync(const QString &path) avoidReadFromDbOnNextSync(path); } -void SyncJournalDb::avoidReadFromDbOnNextSync(const QString &fileName) +void SyncJournalDb::avoidReadFromDbOnNextSync(const QByteArray &fileName) { // Make sure that on the next sync, fileName is not read from the DB but uses the PROPFIND to // get the info from the server @@ -1798,6 +1906,7 @@ void SyncJournalDb::setDataFingerprint(const QByteArray &dataFingerprint) void SyncJournalDb::clearFileTable() { + QMutexLocker lock(&_mutex); SqlQuery query(_db); query.prepare("DELETE FROM metadata;"); query.exec(); diff --git a/src/common/syncjournaldb.h b/src/common/syncjournaldb.h index 8ad2b0852..10324aba7 100644 --- a/src/common/syncjournaldb.h +++ b/src/common/syncjournaldb.h @@ -23,6 +23,7 @@ #include <qmutex.h> #include <QDateTime> #include <QHash> +#include <functional> #include "common/utility.h" #include "common/ownsql.h" @@ -54,7 +55,11 @@ public: static bool maybeMigrateDb(const QString &localPath, const QString &absoluteJournalPath); // To verify that the record could be found check with SyncJournalFileRecord::isValid() - bool getFileRecord(const QString &filename, SyncJournalFileRecord *rec); + bool getFileRecord(const QString &filename, SyncJournalFileRecord *rec) { return getFileRecord(filename.toUtf8(), rec); } + bool getFileRecord(const QByteArray &filename, SyncJournalFileRecord *rec); + bool getFileRecordByInode(quint64 inode, SyncJournalFileRecord *rec); + bool getFileRecordByFileId(const QByteArray &fileId, SyncJournalFileRecord *rec); + bool getFilesBelowPath(const QByteArray &path, const std::function<void(const SyncJournalFileRecord&)> &rowCallback); bool setFileRecord(const SyncJournalFileRecord &record); /// Like setFileRecord, but preserves checksums @@ -72,7 +77,7 @@ public: QString databaseFilePath() const; - static qint64 getPHash(const QString &); + static qint64 getPHash(const QByteArray &); void setErrorBlacklistEntry(const SyncJournalErrorBlacklistRecord &item); void wipeErrorBlacklistEntry(const QString &file); @@ -105,7 +110,7 @@ public: int _chunk; int _transferid; quint64 _size; //currently unused - QDateTime _modtime; + qint64 _modtime; int _errorCount; bool _valid; }; @@ -114,7 +119,7 @@ public: { QString _file; QString _url; - time_t _modtime; + qint64 _modtime; }; DownloadInfo getDownloadInfo(const QString &file); @@ -130,7 +135,8 @@ public: SyncJournalErrorBlacklistRecord errorBlacklistEntry(const QString &); bool deleteStaleErrorBlacklistEntries(const QSet<QString> &keep); - void avoidRenamesOnNextSync(const QString &path); + void avoidRenamesOnNextSync(const QString &path) { avoidRenamesOnNextSync(path.toUtf8()); } + void avoidRenamesOnNextSync(const QByteArray &path); void setPollInfo(const PollInfo &); QVector<PollInfo> getPollInfos(); @@ -164,7 +170,8 @@ public: * _csync_detect_update skip them), the _invalid_ marker will stay and it. And any * child items in the db will be ignored when reading a remote tree from the database. */ - void avoidReadFromDbOnNextSync(const QString &fileName); + void avoidReadFromDbOnNextSync(const QString &fileName) { avoidReadFromDbOnNextSync(fileName.toUtf8()); } + void avoidReadFromDbOnNextSync(const QByteArray &fileName); /** * Ensures full remote discovery happens on the next sync. @@ -233,6 +240,9 @@ private: // NOTE! when adding a query, don't forget to reset it in SyncJournalDb::close QScopedPointer<SqlQuery> _getFileRecordQuery; + QScopedPointer<SqlQuery> _getFileRecordQueryByInode; + QScopedPointer<SqlQuery> _getFileRecordQueryByFileId; + QScopedPointer<SqlQuery> _getFilesBelowPathQuery; QScopedPointer<SqlQuery> _setFileRecordQuery; QScopedPointer<SqlQuery> _setFileRecordChecksumQuery; QScopedPointer<SqlQuery> _setFileRecordLocalMetadataQuery; @@ -258,7 +268,7 @@ private: * It means that they should not be written to the DB in any case since doing * that would write the etag and would void the purpose of avoidReadFromDbOnNextSync */ - QList<QString> _avoidReadFromDbOnNextSyncFilter; + QList<QByteArray> _avoidReadFromDbOnNextSyncFilter; /** The journal mode to use for the db. * diff --git a/src/common/syncjournalfilerecord.cpp b/src/common/syncjournalfilerecord.cpp index c6a77b997..226c25d6c 100644 --- a/src/common/syncjournalfilerecord.cpp +++ b/src/common/syncjournalfilerecord.cpp @@ -52,7 +52,7 @@ bool operator==(const SyncJournalFileRecord &lhs, { return lhs._path == rhs._path && lhs._inode == rhs._inode - && lhs._modtime.toTime_t() == rhs._modtime.toTime_t() + && lhs._modtime == rhs._modtime && lhs._type == rhs._type && lhs._etag == rhs._etag && lhs._fileId == rhs._fileId diff --git a/src/common/syncjournalfilerecord.h b/src/common/syncjournalfilerecord.h index c6008713e..bc34ef135 100644 --- a/src/common/syncjournalfilerecord.h +++ b/src/common/syncjournalfilerecord.h @@ -24,6 +24,7 @@ #include "ocsynclib.h" #include "remotepermissions.h" +#include "common/utility.h" namespace OCC { @@ -50,10 +51,11 @@ public: * It is used in the construction of private links. */ QByteArray numericFileId() const; + QDateTime modDateTime() const { return Utility::qDateTimeFromTime_t(_modtime); } - QString _path; + QByteArray _path; quint64 _inode; - QDateTime _modtime; + qint64 _modtime; int _type; QByteArray _etag; QByteArray _fileId; @@ -94,14 +96,14 @@ public: /// The error category. Sometimes used for special actions. Category _errorCategory; - time_t _lastTryModtime; + qint64 _lastTryModtime; QByteArray _lastTryEtag; /// The last time the operation was attempted (in s since epoch). - time_t _lastTryTime; + qint64 _lastTryTime; /// The number of seconds the file shall be ignored. - time_t _ignoreDuration; + qint64 _ignoreDuration; QString _file; QString _renameTarget; diff --git a/src/csync/CMakeLists.txt b/src/csync/CMakeLists.txt index acb1cc7c2..701405431 100644 --- a/src/csync/CMakeLists.txt +++ b/src/csync/CMakeLists.txt @@ -67,7 +67,6 @@ set(csync_SRCS csync.cpp csync_exclude.cpp csync_log.cpp - csync_statedb.cpp csync_time.c csync_util.cpp csync_misc.cpp diff --git a/src/csync/csync.cpp b/src/csync/csync.cpp index 6c36637b8..d2838884a 100644 --- a/src/csync/csync.cpp +++ b/src/csync/csync.cpp @@ -36,7 +36,6 @@ #include "c_lib.h" #include "csync_private.h" #include "csync_exclude.h" -#include "csync_statedb.h" #include "csync_time.h" #include "csync_util.h" #include "csync_misc.h" @@ -52,7 +51,9 @@ #include "common/c_jhash.h" -csync_s::csync_s(const char *localUri, const char *db_file) { +csync_s::csync_s(const char *localUri, OCC::SyncJournalDb *statedb) + : statedb(statedb) +{ size_t len = 0; /* remove trailing slashes */ @@ -60,8 +61,6 @@ csync_s::csync_s(const char *localUri, const char *db_file) { while(len > 0 && localUri[len - 1] == '/') --len; local.uri = c_strndup(localUri, len); - - statedb.file = c_strdup(db_file); } int csync_update(CSYNC *ctx) { @@ -74,12 +73,6 @@ int csync_update(CSYNC *ctx) { } ctx->status_code = CSYNC_STATUS_OK; - /* Path of database file is set in csync_init */ - if (csync_statedb_load(ctx, ctx->statedb.file, &ctx->statedb.db) < 0) { - rc = -1; - return rc; - } - ctx->status_code = CSYNC_STATUS_OK; csync_memstat_check(); @@ -97,7 +90,7 @@ int csync_update(CSYNC *ctx) { if(ctx->status_code == CSYNC_STATUS_OK) { ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_UPDATE_ERROR); } - goto out; + return rc; } csync_gettime(&finish); @@ -116,7 +109,7 @@ int csync_update(CSYNC *ctx) { if(ctx->status_code == CSYNC_STATUS_OK) { ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_UPDATE_ERROR); } - goto out; + return rc; } csync_gettime(&finish); @@ -130,9 +123,6 @@ int csync_update(CSYNC *ctx) { ctx->status |= CSYNC_STATUS_UPDATE; rc = 0; - -out: - csync_statedb_close(ctx); return rc; } @@ -149,11 +139,6 @@ int csync_reconcile(CSYNC *ctx) { /* Reconciliation for local replica */ csync_gettime(&start); - if (csync_statedb_load(ctx, ctx->statedb.file, &ctx->statedb.db) < 0) { - rc = -1; - return rc; - } - ctx->current = LOCAL_REPLICA; rc = csync_reconcile_updates(ctx); @@ -168,7 +153,7 @@ int csync_reconcile(CSYNC *ctx) { if (!CSYNC_STATUS_IS_OK(ctx->status_code)) { ctx->status_code = csync_errno_to_status( errno, CSYNC_STATUS_RECONCILE_ERROR ); } - goto out; + return rc; } /* Reconciliation for remote replica */ @@ -188,16 +173,13 @@ int csync_reconcile(CSYNC *ctx) { if (!CSYNC_STATUS_IS_OK(ctx->status_code)) { ctx->status_code = csync_errno_to_status(errno, CSYNC_STATUS_RECONCILE_ERROR ); } - goto out; + return rc; } ctx->status |= CSYNC_STATUS_RECONCILE; rc = 0; - -out: - csync_statedb_close(ctx); - return 0; + return rc; } /* @@ -326,13 +308,6 @@ int csync_s::reinitialize() { status_code = CSYNC_STATUS_OK; - if (statedb.db != NULL - && csync_statedb_close(this) < 0) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "ERR: closing of statedb failed."); - rc = -1; - } - statedb.db = NULL; - remote.read_from_db = 0; read_remote_from_db = true; db_is_empty = false; @@ -348,13 +323,6 @@ int csync_s::reinitialize() { } csync_s::~csync_s() { - if (statedb.db != NULL - && csync_statedb_close(this) < 0) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "ERR: closing of statedb failed."); - } - statedb.db = NULL; - - SAFE_FREE(statedb.file); SAFE_FREE(local.uri); SAFE_FREE(error_string); } diff --git a/src/csync/csync.h b/src/csync/csync.h index ef2d3bada..4dfa989f1 100644 --- a/src/csync/csync.h +++ b/src/csync/csync.h @@ -34,6 +34,8 @@ #include "std/c_private.h" #include "ocsynclib.h" +#include "common/syncjournalfilerecord.h" + #include <sys/stat.h> #include <stdbool.h> #include <stdint.h> @@ -200,6 +202,22 @@ struct OCSYNC_EXPORT csync_file_stat_s { , error_status(CSYNC_STATUS_OK) , instruction(CSYNC_INSTRUCTION_NONE) { } + + static std::unique_ptr<csync_file_stat_t> fromSyncJournalFileRecord(const OCC::SyncJournalFileRecord &rec) + { + std::unique_ptr<csync_file_stat_t> st(new csync_file_stat_t); + st->path = rec._path; + st->inode = rec._inode; + st->modtime = rec._modtime; + st->type = static_cast<csync_ftw_type_e>(rec._type); + st->etag = rec._etag; + st->file_id = rec._fileId; + st->remotePerm = rec._remotePerm; + st->size = rec._fileSize; + st->has_ignored_files = rec._serverHasIgnoredFiles; + st->checksumHeader = rec._checksumHeader; + return st; + } }; /** diff --git a/src/csync/csync_private.h b/src/csync/csync_private.h index f3398f7a7..c898eeabe 100644 --- a/src/csync/csync_private.h +++ b/src/csync/csync_private.h @@ -39,6 +39,7 @@ #include <sqlite3.h> #include <map> +#include "common/syncjournaldb.h" #include "config_csync.h" #include "std/c_lib.h" #include "std/c_private.h" @@ -103,17 +104,7 @@ struct OCSYNC_EXPORT csync_s { } callbacks; c_strlist_t *excludes = nullptr; - struct { - char *file = nullptr; - sqlite3 *db = nullptr; - bool exists = false; - - sqlite3_stmt* by_hash_stmt = nullptr; - sqlite3_stmt* by_fileid_stmt = nullptr; - sqlite3_stmt* by_inode_stmt = nullptr; - - int lastReturnValue; - } statedb; + OCC::SyncJournalDb *statedb; struct { std::map<QByteArray, QByteArray> folder_renamed_to; // map from->to @@ -159,7 +150,7 @@ struct OCSYNC_EXPORT csync_s { bool ignore_hidden_files = true; - csync_s(const char *localUri, const char *db_file); + csync_s(const char *localUri, OCC::SyncJournalDb *statedb); ~csync_s(); int reinitialize(); diff --git a/src/csync/csync_reconcile.cpp b/src/csync/csync_reconcile.cpp index 76fd08408..2fdaa7f38 100644 --- a/src/csync/csync_reconcile.cpp +++ b/src/csync/csync_reconcile.cpp @@ -24,7 +24,6 @@ #include "csync_private.h" #include "csync_reconcile.h" #include "csync_util.h" -#include "csync_statedb.h" #include "csync_rename.h" #include "common/c_jhash.h" #include "common/asserts.h" @@ -104,8 +103,6 @@ static bool _csync_is_collision_safe_hash(const char *checksum_header) * source and the destination, have been changed, the newer file wins. */ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { - std::unique_ptr<csync_file_stat_t> tmp; - csync_s::FileMap *other_tree = nullptr; /* we need the opposite tree! */ @@ -153,35 +150,34 @@ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { } cur->instruction = CSYNC_INSTRUCTION_REMOVE; break; - case CSYNC_INSTRUCTION_EVAL_RENAME: + case CSYNC_INSTRUCTION_EVAL_RENAME: { + OCC::SyncJournalFileRecord base; if(ctx->current == LOCAL_REPLICA ) { /* use the old name to find the "other" node */ - tmp = csync_statedb_get_stat_by_inode(ctx, cur->inode); + ctx->statedb->getFileRecordByInode(cur->inode, &base); qCDebug(lcReconcile, "Finding opposite temp through inode %" PRIu64 ": %s", - cur->inode, tmp ? "true":"false"); + cur->inode, base.isValid() ? "true":"false"); } else { ASSERT( ctx->current == REMOTE_REPLICA ); - tmp = csync_statedb_get_stat_by_file_id(ctx, cur->file_id); + ctx->statedb->getFileRecordByFileId(cur->file_id, &base); qCDebug(lcReconcile, "Finding opposite temp through file ID %s: %s", - cur->file_id.constData(), tmp ? "true":"false"); + cur->file_id.constData(), base.isValid() ? "true":"false"); } - if( tmp ) { - if( !tmp->path.isEmpty() ) { - /* First, check that the file is NOT in our tree (another file with the same name was added) */ - csync_s::FileMap *our_tree = ctx->current == REMOTE_REPLICA ? &ctx->remote.files : &ctx->local.files; - if (our_tree->findFile(tmp->path)) { - qCDebug(lcReconcile, "Origin found in our tree : %s", tmp->path.constData()); - } else { + if( base.isValid() ) { + /* First, check that the file is NOT in our tree (another file with the same name was added) */ + csync_s::FileMap *our_tree = ctx->current == REMOTE_REPLICA ? &ctx->remote.files : &ctx->local.files; + if (our_tree->findFile(base._path)) { + qCDebug(lcReconcile, "Origin found in our tree : %s", base._path.constData()); + } else { /* Find the temporar file in the other tree. * If the renamed file could not be found in the opposite tree, that is because it * is not longer existing there, maybe because it was renamed or deleted. * The journal is cleaned up later after propagation. */ - other = other_tree->findFile(tmp->path); + other = other_tree->findFile(base._path); qCDebug(lcReconcile, "Temporary opposite (%s) %s", - tmp->path.constData() , other ? "found": "not found" ); - } + base._path.constData() , other ? "found": "not found" ); } if(!other) { @@ -213,9 +209,9 @@ static int _csync_merge_algorithm_visitor(csync_file_stat_t *cur, CSYNC * ctx) { cur->instruction = CSYNC_INSTRUCTION_NONE; other->instruction = CSYNC_INSTRUCTION_SYNC; } - } - + } break; + } default: break; } diff --git a/src/csync/csync_statedb.cpp b/src/csync/csync_statedb.cpp deleted file mode 100644 index b489b6789..000000000 --- a/src/csync/csync_statedb.cpp +++ /dev/null @@ -1,627 +0,0 @@ -/* - * libcsync -- a library to sync a directory with another - * - * Copyright (c) 2008-2013 by Andreas Schneider <asn@cryptomilk.org> - * Copyright (c) 2012-2013 by Klaas Freitag <freitag@owncloud.com> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#include "config_csync.h" - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include <assert.h> -#include <sqlite3.h> -#include <stdio.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> -#include <errno.h> - -#include "c_lib.h" -#include "csync_private.h" -#include "csync_statedb.h" -#include "csync_util.h" -#include "csync_misc.h" -#include "csync_exclude.h" - -#include "c_string.h" -#include "common/c_jhash.h" -#include "c_utf8.h" -#include "csync_time.h" - -#define CSYNC_LOG_CATEGORY_NAME "csync.statedb" -#include "csync_log.h" -#include "csync_rename.h" - -// Needed for PRIu64 on MinGW in C++ mode. -#define __STDC_FORMAT_MACROS -#include <inttypes.h> - -#define BUF_SIZE 16 - -#define sqlite_open(A, B) sqlite3_open_v2(A,B, SQLITE_OPEN_READONLY+SQLITE_OPEN_NOMUTEX, NULL) - -#define SQLTM_TIME 150 -#define SQLTM_COUNT 10 - -#define SQLITE_BUSY_HANDLED(F) if(1) { \ - int n = 0; \ - do { rc = F ; \ - if( (rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED) ) { \ - n++; \ - csync_sleep(SQLTM_TIME); \ - } \ - }while( (n < SQLTM_COUNT) && ((rc == SQLITE_BUSY) || (rc == SQLITE_LOCKED))); \ - } - - -void csync_set_statedb_exists(CSYNC *ctx, int val) { - ctx->statedb.exists = val; -} - -int csync_get_statedb_exists(CSYNC *ctx) { - return ctx->statedb.exists; -} - -static int _csync_check_db_integrity(sqlite3 *db) { - c_strlist_t *result = NULL; - int rc = -1; - - result = csync_statedb_query(db, "PRAGMA quick_check;"); - if (result != NULL) { - /* There is a result */ - if (result->count > 0) { - if (c_streq(result->vector[0], "ok")) { - rc = 0; - } - } - c_strlist_destroy(result); - } - - if( sqlite3_threadsafe() == 0 ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "* WARNING: SQLite module is not threadsafe!"); - rc = -1; - } - - return rc; -} - -static int _csync_statedb_is_empty(sqlite3 *db) { - c_strlist_t *result = NULL; - int rc = 0; - - result = csync_statedb_query(db, "SELECT COUNT(phash) FROM metadata LIMIT 1 OFFSET 0;"); - if (result == NULL) { - rc = 1; - } - c_strlist_destroy(result); - - return rc; -} - -#ifndef NDEBUG -static void sqlite_profile( void *x, const char* sql, sqlite3_uint64 time) -{ - (void)x; - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, - "_SQL_ %s: %llu", sql, time); - -} -#endif - -int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb) { - int rc = -1; - c_strlist_t *result = NULL; - sqlite3 *db = NULL; - - if( !ctx ) { - return -1; - } - - if (ctx->statedb.db) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "ERR: DB already open"); - ctx->status_code = CSYNC_STATUS_PARAM_ERROR; - return -1; - } - - ctx->statedb.lastReturnValue = SQLITE_OK; - - /* Openthe database */ - if (sqlite_open(statedb, &db) != SQLITE_OK) { - const char *errmsg= sqlite3_errmsg(ctx->statedb.db); - CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "ERR: Failed to sqlite3 open statedb - bail out: %s.", - errmsg ? errmsg : "<no sqlite3 errormsg>"); - - rc = -1; - ctx->status_code = CSYNC_STATUS_STATEDB_LOAD_ERROR; - goto out; - } - - if (_csync_check_db_integrity(db) != 0) { - const char *errmsg= sqlite3_errmsg(db); - CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "ERR: sqlite3 integrity check failed - bail out: %s.", - errmsg ? errmsg : "<no sqlite3 errormsg>"); - rc = -1; - ctx->status_code = CSYNC_STATUS_STATEDB_CORRUPTED; - goto out; - } - - if (_csync_statedb_is_empty(db)) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_NOTICE, "statedb contents doesn't exist"); - csync_set_statedb_exists(ctx, 0); - } else { - csync_set_statedb_exists(ctx, 1); - } - - /* Print out the version */ - // - result = csync_statedb_query(db, "SELECT sqlite_version();"); - if (result && result->count >= 1) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "sqlite3 version \"%s\"", *result->vector); - } - c_strlist_destroy(result); - - /* optimization for speeding up SQLite */ - result = csync_statedb_query(db, "PRAGMA synchronous = NORMAL;"); - c_strlist_destroy(result); - result = csync_statedb_query(db, "PRAGMA case_sensitive_like = ON;"); - c_strlist_destroy(result); - - /* set a busy handler with 5 seconds timeout */ - sqlite3_busy_timeout(db, 5000); - -#ifndef NDEBUG - sqlite3_profile(db, sqlite_profile, 0 ); -#endif - *pdb = db; - - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Success"); - - return 0; -out: - sqlite3_close(db); - return rc; -} - -int csync_statedb_close(CSYNC *ctx) { - int rc = 0; - - if (!ctx) { - return -1; - } - - /* deallocate query resources */ - if( ctx->statedb.by_fileid_stmt ) { - sqlite3_finalize(ctx->statedb.by_fileid_stmt); - ctx->statedb.by_fileid_stmt = NULL; - } - if( ctx->statedb.by_hash_stmt ) { - sqlite3_finalize(ctx->statedb.by_hash_stmt); - ctx->statedb.by_hash_stmt = NULL; - } - if( ctx->statedb.by_inode_stmt) { - sqlite3_finalize(ctx->statedb.by_inode_stmt); - ctx->statedb.by_inode_stmt = NULL; - } - - ctx->statedb.lastReturnValue = SQLITE_OK; - - int sr = sqlite3_close(ctx->statedb.db); - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "sqlite3_close=%d", sr); - - ctx->statedb.db = 0; - - return rc; -} - -#define METADATA_QUERY \ - "path, inode, modtime, type, md5, fileid, remotePerm, " \ - "filesize, ignoredChildrenRemote, " \ - "contentchecksumtype.name || ':' || contentChecksum " \ - "FROM metadata " \ - "LEFT JOIN checksumtype as contentchecksumtype ON metadata.contentChecksumTypeId == contentchecksumtype.id" - -// This funciton parses a line from the metadata table into the given csync_file_stat -// structure which it is also allocating. -// Note that this function calls laso sqlite3_step to actually get the info from db and -// returns the sqlite return type. -static int _csync_file_stat_from_metadata_table( std::unique_ptr<csync_file_stat_t> &st, sqlite3_stmt *stmt ) -{ - int rc = SQLITE_ERROR; - - if( ! stmt ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Fatal: Statement is NULL."); - return SQLITE_ERROR; - } - - // Callers should all use METADATA_QUERY for their column list. - assert(sqlite3_column_count(stmt) == 10); - - SQLITE_BUSY_HANDLED( sqlite3_step(stmt) ); - - if( rc == SQLITE_ROW ) { - st.reset(new csync_file_stat_t); - - st->path = (char*)sqlite3_column_text(stmt, 0); - st->inode = sqlite3_column_int64(stmt, 1); - st->modtime = strtoul((char*)sqlite3_column_text(stmt, 2), NULL, 10); - st->type = static_cast<enum csync_ftw_type_e>(sqlite3_column_int(stmt, 3)); - st->etag = (char*)sqlite3_column_text(stmt, 4); - st->file_id = (char*)sqlite3_column_text(stmt, 5); - const char *permStr = (char *)sqlite3_column_text(stmt, 6); - // If permStr is empty, construct a null RemotePermissions. We make sure that non-null - // permissions are never empty in RemotePermissions.toString() - st->remotePerm = permStr && *permStr ? OCC::RemotePermissions(permStr) : OCC::RemotePermissions(); - st->size = sqlite3_column_int64(stmt, 7); - st->has_ignored_files = sqlite3_column_int(stmt, 8); - st->checksumHeader = (char *)sqlite3_column_text(stmt, 9); - } else { - if( rc != SQLITE_DONE ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, "Query results in %d", rc); - } - } - return rc; -} - -/* caller must free the memory */ -std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_path(CSYNC *ctx, const QByteArray &path) -{ - std::unique_ptr<csync_file_stat_t> st; - int rc; - - if( !ctx || ctx->db_is_empty ) { - return NULL; - } - - if( ctx->statedb.by_hash_stmt == NULL ) { - const char *hash_query = "SELECT " METADATA_QUERY " WHERE phash=?1"; - - SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, hash_query, strlen(hash_query), &ctx->statedb.by_hash_stmt, NULL)); - ctx->statedb.lastReturnValue = rc; - if( rc != SQLITE_OK ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for hash query."); - return NULL; - } - } - - if( ctx->statedb.by_hash_stmt == NULL ) { - return NULL; - } - - uint64_t phash = c_jhash64((const uint8_t*)path.constData(), path.size(), 0); - sqlite3_bind_int64(ctx->statedb.by_hash_stmt, 1, (long long signed int)phash); - - rc = _csync_file_stat_from_metadata_table(st, ctx->statedb.by_hash_stmt); - ctx->statedb.lastReturnValue = rc; - if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata: %d!", rc); - } - sqlite3_reset(ctx->statedb.by_hash_stmt); - - return st; -} - -std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_file_id(CSYNC *ctx, - const char *file_id ) { - std::unique_ptr<csync_file_stat_t> st; - int rc = 0; - - if (!file_id) { - return 0; - } - if (c_streq(file_id, "")) { - return 0; - } - - if( !ctx || ctx->db_is_empty ) { - return NULL; - } - - if( ctx->statedb.by_fileid_stmt == NULL ) { - const char *query = "SELECT " METADATA_QUERY " WHERE fileid=?1"; - - SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, query, strlen(query), &ctx->statedb.by_fileid_stmt, NULL)); - ctx->statedb.lastReturnValue = rc; - if( rc != SQLITE_OK ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for file id query."); - return NULL; - } - } - - /* bind the query value */ - sqlite3_bind_text(ctx->statedb.by_fileid_stmt, 1, file_id, -1, SQLITE_STATIC); - - rc = _csync_file_stat_from_metadata_table(st, ctx->statedb.by_fileid_stmt); - ctx->statedb.lastReturnValue = rc; - if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata: %d!", rc); - } - // clear the resources used by the statement. - sqlite3_reset(ctx->statedb.by_fileid_stmt); - - return st; -} - -/* caller must free the memory */ -std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_inode(CSYNC *ctx, - uint64_t inode) -{ - std::unique_ptr<csync_file_stat_t> st; - int rc; - - if (!inode) { - return NULL; - } - - if( !ctx || ctx->db_is_empty ) { - return NULL; - } - - if( ctx->statedb.by_inode_stmt == NULL ) { - const char *inode_query = "SELECT " METADATA_QUERY " WHERE inode=?1"; - - SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, inode_query, strlen(inode_query), &ctx->statedb.by_inode_stmt, NULL)); - ctx->statedb.lastReturnValue = rc; - if( rc != SQLITE_OK ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for inode query."); - return NULL; - } - } - - if( ctx->statedb.by_inode_stmt == NULL ) { - return NULL; - } - - sqlite3_bind_int64(ctx->statedb.by_inode_stmt, 1, (long long signed int)inode); - - rc = _csync_file_stat_from_metadata_table(st, ctx->statedb.by_inode_stmt); - ctx->statedb.lastReturnValue = rc; - if( !(rc == SQLITE_ROW || rc == SQLITE_DONE) ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Could not get line from metadata by inode: %d!", rc); - } - sqlite3_reset(ctx->statedb.by_inode_stmt); - - return st; -} - -int csync_statedb_get_below_path( CSYNC *ctx, const char *path ) { - int rc; - sqlite3_stmt *stmt = NULL; - int64_t cnt = 0; - - if( !path ) { - return -1; - } - - if( !ctx || ctx->db_is_empty ) { - return -1; - } - - /* Select the entries for anything that starts with (path+'/') - * In other words, anything that is between path+'/' and path+'0', - * (because '0' follows '/' in ascii) - */ - const char *below_path_query = "SELECT " METADATA_QUERY " WHERE path > (?||'/') AND path < (?||'0') ORDER BY path||'/' ASC"; - SQLITE_BUSY_HANDLED(sqlite3_prepare_v2(ctx->statedb.db, below_path_query, -1, &stmt, NULL)); - ctx->statedb.lastReturnValue = rc; - if( rc != SQLITE_OK ) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "WRN: Unable to create stmt for below path query."); - return -1; - } - - if (stmt == NULL) { - return -1; - } - - sqlite3_bind_text(stmt, 1, path, -1, SQLITE_STATIC); - sqlite3_bind_text(stmt, 2, path, -1, SQLITE_STATIC); - - cnt = 0; - - ctx->statedb.lastReturnValue = rc; - do { - std::unique_ptr<csync_file_stat_t> st; - - rc = _csync_file_stat_from_metadata_table(st, stmt); - if( st ) { - /* When selective sync is used, the database may have subtrees with a parent - * whose etag (md5) is _invalid_. These are ignored and shall not appear in the - * remote tree. - * Sometimes folders that are not ignored by selective sync get marked as - * _invalid_, but that is not a problem as the next discovery will retrieve - * their correct etags again and we don't run into this case. - */ - if( st->etag == "_invalid_") { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s selective sync excluded", st->path.constData()); - QByteArray skipbase = st->path; - skipbase += '/'; - - /* Skip over all entries with the same base path. Note that this depends - * strongly on the ordering of the retrieved items. */ - do { - st.reset(); - rc = _csync_file_stat_from_metadata_table(st, stmt); - if( st ) { - if( !st->path.startsWith(skipbase) ) - break; - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s selective sync excluded because the parent is", st->path.constData()); - } - } while( rc == SQLITE_ROW ); - - /* End of data? */ - if( rc != SQLITE_ROW || !st ) { - continue; - } - } - - /* Check for exclusion from the tree. - * Note that this is only a safety net in case the ignore list changes - * without a full remote discovery being triggered. */ - CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type); - if (excluded != CSYNC_NOT_EXCLUDED) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "%s excluded (%d)", st->path.constData(), excluded); - - if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE - || excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { - st.reset(); - continue; - } - - st->instruction = CSYNC_INSTRUCTION_IGNORE; - } - - /* store into result list. */ - QByteArray path = st->path; - ctx->remote.files[path] = std::move(st); - cnt++; - } - } while( rc == SQLITE_ROW ); - - ctx->statedb.lastReturnValue = rc; - if( rc != SQLITE_DONE ) { - ctx->status_code = CSYNC_STATUS_STATEDB_LOAD_ERROR; - } else { - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "%" PRId64 " entries read below path %s from db.", cnt, path); - } - sqlite3_finalize(stmt); - - return 0; -} - -/* query the statedb, caller must free the memory */ -c_strlist_t *csync_statedb_query(sqlite3 *db, - const char *statement) { - int err = SQLITE_OK; - int rc = SQLITE_OK; - size_t i = 0; - size_t busy_count = 0; - size_t retry_count = 0; - size_t column_count = 0; - sqlite3_stmt *stmt; - const char *tail = NULL; - const char *field = NULL; - c_strlist_t *result = NULL; - int row = 0; - - do { - /* compile SQL program into a virtual machine, reattempteing if busy */ - do { - if (busy_count) { - csync_sleep(100); - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "sqlite3_prepare: BUSY counter: %zu", busy_count); - } - err = sqlite3_prepare(db, statement, -1, &stmt, &tail); - } while (err == SQLITE_BUSY && busy_count ++ < 120); - - if (err != SQLITE_OK) { - if (err == SQLITE_BUSY) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Gave up waiting for lock to clear"); - } - CSYNC_LOG(CSYNC_LOG_PRIORITY_WARN, - "sqlite3_compile error: %s - on query %s", - sqlite3_errmsg(db), statement); - break; - } else { - busy_count = 0; - column_count = sqlite3_column_count(stmt); - - /* execute virtual machine by iterating over rows */ - for(;;) { - err = sqlite3_step(stmt); - - if (err == SQLITE_BUSY) { - if (busy_count++ > 120) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "Busy counter has reached its maximum. Aborting this sql statement"); - break; - } - csync_sleep(100); - CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_step: BUSY counter: %zu", busy_count); - continue; - } - - if (err == SQLITE_MISUSE) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "sqlite3_step: MISUSE!!"); - } - - if (err == SQLITE_DONE) { - if (result == NULL) { - result = c_strlist_new(1); - } - break; - } - - if (err == SQLITE_ERROR) { - break; - } - - row++; - if( result ) { - result = c_strlist_expand(result, row*column_count); - } else { - result = c_strlist_new(column_count); - } - - if (result == NULL) { - return NULL; - } - - /* iterate over columns */ - for (i = 0; i < column_count; i++) { - field = (const char *) sqlite3_column_text(stmt, i); - if (!field) - field = ""; - // CSYNC_LOG(CSYNC_LOG_PRIORITY_TRACE, "sqlite3_column_text: %s", field); - if (c_strlist_add(result, field) < 0) { - c_strlist_destroy(result); - return NULL; - } - } - } /* end infinite for loop */ - - /* deallocate vm resources */ - rc = sqlite3_finalize(stmt); - - if (err != SQLITE_DONE && rc != SQLITE_SCHEMA) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "sqlite_step error: %s - on query: %s", sqlite3_errmsg(db), statement); - if (result != NULL) { - c_strlist_destroy(result); - } - return NULL; - } - - if (rc == SQLITE_SCHEMA) { - retry_count ++; - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "SQLITE_SCHEMA error occurred on query: %s", statement); - if (retry_count < 10) { - CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "Retrying now."); - } else { - CSYNC_LOG(CSYNC_LOG_PRIORITY_ERROR, "RETRY count has reached its maximum. Aborting statement: %s", statement); - if (result != NULL) { - c_strlist_destroy(result); - } - result = c_strlist_new(1); - } - } - } - } while (rc == SQLITE_SCHEMA && retry_count < 10); - - return result; -} - -/* vim: set ts=8 sw=2 et cindent: */ diff --git a/src/csync/csync_statedb.h b/src/csync/csync_statedb.h deleted file mode 100644 index 28c1cc823..000000000 --- a/src/csync/csync_statedb.h +++ /dev/null @@ -1,99 +0,0 @@ -/* - * libcsync -- a library to sync a directory with another - * - * Copyright (c) 2008-2013 by Andreas Schneider <asn@cryptomilk.org> - * Copyright (c) 2012-2013 by Klaas Freitag <freitag@owncloud.com> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -/** - * @file csync_private.h - * - * @brief Private interface of csync - * - * @defgroup csyncstatedbInternals csync statedb internals - * @ingroup csyncInternalAPI - * - * @{ - */ - -#ifndef _CSYNC_STATEDB_H -#define _CSYNC_STATEDB_H - -#include "c_lib.h" -#include "csync_private.h" - -void csync_set_statedb_exists(CSYNC *ctx, int val); - -int csync_get_statedb_exists(CSYNC *ctx); - -/** - * @brief Load the statedb. - * - * This function tries to load the statedb. If it doesn't exists it creates - * the sqlite3 database, but doesn't create the tables. This will be done when - * csync gets destroyed. - * - * @param ctx The csync context. - * @param statedb Path to the statedb file (sqlite3 db). - * - * @return 0 on success, less than 0 if an error occurred with errno set. - */ -OCSYNC_EXPORT int csync_statedb_load(CSYNC *ctx, const char *statedb, sqlite3 **pdb); - -OCSYNC_EXPORT int csync_statedb_close(CSYNC *ctx); - -OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_path(CSYNC *ctx, const QByteArray &path); - -OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_inode(CSYNC *ctx, uint64_t inode); - -OCSYNC_EXPORT std::unique_ptr<csync_file_stat_t> csync_statedb_get_stat_by_file_id(CSYNC *ctx, const char *file_id); - -/** - * @brief Query all files metadata inside and below a path. - * @param ctx The csync context. - * @param path The path. - * - * This function queries all metadata of all files inside or below the - * given path. The result is a linear string list with a multiple of 9 - * entries. For each result file there are 9 strings which are phash, - * path, inode, uid, gid, mode, modtime, type and md5 (unique id). - * - * Note that not only the files in the given path are part of the result - * but also the files in directories below the given path. Ie. if the - * parameter path is /home/kf/test, we have /home/kf/test/file.txt in - * the result but also /home/kf/test/homework/another_file.txt - * - * @return A stringlist containing a multiple of 9 entries. - */ -int csync_statedb_get_below_path(CSYNC *ctx, const char *path); - -/** - * @brief A generic statedb query. - * - * @param ctx The csync context. - * @param statement The SQL statement to execute - * - * @return A stringlist of the entries of a column. An emtpy stringlist if - * nothing has been found. NULL on error. - */ -c_strlist_t *csync_statedb_query(sqlite3 *db, const char *statement); - -/** - * }@ - */ -#endif /* _CSYNC_STATEDB_H */ -/* vim: set ft=c.doxygen ts=8 sw=2 et cindent: */ diff --git a/src/csync/csync_update.cpp b/src/csync/csync_update.cpp index c42eaa928..8f88c324b 100644 --- a/src/csync/csync_update.cpp +++ b/src/csync/csync_update.cpp @@ -35,7 +35,6 @@ #include "csync_private.h" #include "csync_exclude.h" -#include "csync_statedb.h" #include "csync_update.h" #include "csync_util.h" #include "csync_misc.h" @@ -74,10 +73,6 @@ static bool _csync_sameextension(const char *p1, const char *p2) { } #endif -static bool _last_db_return_error(CSYNC* ctx) { - return ctx->statedb.lastReturnValue != SQLITE_OK && ctx->statedb.lastReturnValue != SQLITE_DONE && ctx->statedb.lastReturnValue != SQLITE_ROW; -} - static QByteArray _rel_to_abs(CSYNC* ctx, const QByteArray &relativePath) { return QByteArray() % const_cast<const char *>(ctx->local.uri) % '/' % relativePath; } @@ -111,7 +106,7 @@ static bool _csync_mtime_equal(time_t a, time_t b) * See doc/dev/sync-algorithm.md for an overview. */ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> fs) { - std::unique_ptr<csync_file_stat_t> tmp; + OCC::SyncJournalFileRecord base; CSYNC_EXCLUDE_TYPE excluded; if (fs == NULL) { @@ -173,48 +168,46 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f * renamed, the db gets queried by the inode of the file as that one * does not change on rename. */ - tmp = csync_statedb_get_stat_by_path(ctx, fs->path); - - if(_last_db_return_error(ctx)) { + if(!ctx->statedb->getFileRecord(fs->path, &base)) { ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; return -1; } - if(tmp && tmp->path == fs->path ) { /* there is an entry in the database */ + if(base.isValid()) { /* there is an entry in the database */ /* we have an update! */ qCInfo(lcUpdate, "Database entry found, compare: %" PRId64 " <-> %" PRId64 ", etag: %s <-> %s, inode: %" PRId64 " <-> %" PRId64 ", size: %" PRId64 " <-> %" PRId64 ", perms: %x <-> %x, ignore: %d", - ((int64_t) fs->modtime), ((int64_t) tmp->modtime), - fs->etag.constData(), tmp->etag.constData(), (uint64_t) fs->inode, (uint64_t) tmp->inode, - (uint64_t) fs->size, (uint64_t) tmp->size, *reinterpret_cast<short*>(&fs->remotePerm), *reinterpret_cast<short*>(&tmp->remotePerm), tmp->has_ignored_files ); - if (ctx->current == REMOTE_REPLICA && fs->etag != tmp->etag) { + ((int64_t) fs->modtime), ((int64_t) base._modtime), + fs->etag.constData(), base._etag.constData(), (uint64_t) fs->inode, (uint64_t) base._inode, + (uint64_t) fs->size, (uint64_t) base._fileSize, *reinterpret_cast<short*>(&fs->remotePerm), *reinterpret_cast<short*>(&base._remotePerm), base._serverHasIgnoredFiles ); + if (ctx->current == REMOTE_REPLICA && fs->etag != base._etag) { fs->instruction = CSYNC_INSTRUCTION_EVAL; // Preserve the EVAL flag later on if the type has changed. - if (tmp->type != fs->type) { + if (base._type != fs->type) { fs->child_modified = true; } goto out; } if (ctx->current == LOCAL_REPLICA && - (!_csync_mtime_equal(fs->modtime, tmp->modtime) + (!_csync_mtime_equal(fs->modtime, base._modtime) // zero size in statedb can happen during migration - || (tmp->size != 0 && fs->size != tmp->size))) { + || (base._fileSize != 0 && fs->size != base._fileSize))) { // Checksum comparison at this stage is only enabled for .eml files, // check #4754 #4755 bool isEmlFile = csync_fnmatch("*.eml", fs->path, FNM_CASEFOLD) == 0; - if (isEmlFile && fs->size == tmp->size && !tmp->checksumHeader.isEmpty()) { + if (isEmlFile && fs->size == base._fileSize && !base._checksumHeader.isEmpty()) { if (ctx->callbacks.checksum_hook) { fs->checksumHeader = ctx->callbacks.checksum_hook( - _rel_to_abs(ctx, fs->path), tmp->checksumHeader, + _rel_to_abs(ctx, fs->path), base._checksumHeader, ctx->callbacks.checksum_userdata); } bool checksumIdentical = false; if (!fs->checksumHeader.isEmpty()) { - checksumIdentical = fs->checksumHeader == tmp->checksumHeader; + checksumIdentical = fs->checksumHeader == base._checksumHeader; } if (checksumIdentical) { qCDebug(lcUpdate, "NOTE: Checksums are identical, file did not actually change: %s", fs->path.constData()); @@ -224,16 +217,16 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f } // Preserve the EVAL flag later on if the type has changed. - if (tmp->type != fs->type) { + if (base._type != fs->type) { fs->child_modified = true; } fs->instruction = CSYNC_INSTRUCTION_EVAL; goto out; } - bool metadata_differ = (ctx->current == REMOTE_REPLICA && (fs->file_id != tmp->file_id - || fs->remotePerm != tmp->remotePerm)) - || (ctx->current == LOCAL_REPLICA && fs->inode != tmp->inode); + bool metadata_differ = (ctx->current == REMOTE_REPLICA && (fs->file_id != base._fileId + || fs->remotePerm != base._remotePerm)) + || (ctx->current == LOCAL_REPLICA && fs->inode != base._inode); if (fs->type == CSYNC_FTW_TYPE_DIR && ctx->current == REMOTE_REPLICA && !metadata_differ && ctx->read_remote_from_db) { /* If both etag and file id are equal for a directory, read all contents from @@ -248,7 +241,7 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f * that so that the reconciler can make advantage of. */ if( ctx->current == REMOTE_REPLICA ) { - fs->has_ignored_files = tmp->has_ignored_files; + fs->has_ignored_files = base._serverHasIgnoredFiles; } if (metadata_differ) { /* file id or permissions has changed. Which means we need to update them in the DB. */ @@ -262,9 +255,8 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f if (ctx->current == LOCAL_REPLICA) { qCDebug(lcUpdate, "Checking for rename based on inode # %" PRId64 "", (uint64_t) fs->inode); - tmp = csync_statedb_get_stat_by_inode(ctx, fs->inode); - - if(_last_db_return_error(ctx)) { + OCC::SyncJournalFileRecord base; + if(!ctx->statedb->getFileRecordByInode(fs->inode, &base)) { ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; return -1; } @@ -273,23 +265,23 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f fs->instruction = CSYNC_INSTRUCTION_NEW; bool isRename = - tmp && tmp->inode == fs->inode && tmp->type == fs->type - && (tmp->modtime == fs->modtime || fs->type == CSYNC_FTW_TYPE_DIR) + base.isValid() && base._inode == fs->inode && base._type == fs->type + && (base._modtime == fs->modtime || fs->type == CSYNC_FTW_TYPE_DIR) #ifdef NO_RENAME_EXTENSION - && _csync_sameextension(tmp->path, fs->path) + && _csync_sameextension(base._path, fs->path) #endif ; // Verify the checksum where possible - if (isRename && !tmp->checksumHeader.isEmpty() && ctx->callbacks.checksum_hook + if (isRename && !base._checksumHeader.isEmpty() && ctx->callbacks.checksum_hook && fs->type == CSYNC_FTW_TYPE_FILE) { fs->checksumHeader = ctx->callbacks.checksum_hook( - _rel_to_abs(ctx, fs->path), tmp->checksumHeader, + _rel_to_abs(ctx, fs->path), base._checksumHeader, ctx->callbacks.checksum_userdata); if (!fs->checksumHeader.isEmpty()) { - qCDebug(lcUpdate, "checking checksum of potential rename %s %s <-> %s", fs->path.constData(), fs->checksumHeader.constData(), tmp->checksumHeader.constData()); - isRename = fs->checksumHeader == tmp->checksumHeader; + qCDebug(lcUpdate, "checking checksum of potential rename %s %s <-> %s", fs->path.constData(), fs->checksumHeader.constData(), base._checksumHeader.constData()); + isRename = fs->checksumHeader == base._checksumHeader; } } @@ -298,31 +290,30 @@ static int _csync_detect_update(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> f /* inode found so the file has been renamed */ fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; if (fs->type == CSYNC_FTW_TYPE_DIR) { - csync_rename_record(ctx, tmp->path, fs->path); + csync_rename_record(ctx, base._path, fs->path); } } goto out; } else { /* Remote Replica Rename check */ - tmp = csync_statedb_get_stat_by_file_id(ctx, fs->file_id); - - if(_last_db_return_error(ctx)) { + OCC::SyncJournalFileRecord base; + if(!ctx->statedb->getFileRecordByFileId(fs->file_id, &base)) { ctx->status_code = CSYNC_STATUS_UNSUCCESSFUL; return -1; } - if(tmp ) { /* tmp existing at all */ - if (tmp->type != fs->type) { + if (base.isValid()) { /* tmp existing at all */ + if (base._type != fs->type) { qCWarning(lcUpdate, "file types different is not!"); fs->instruction = CSYNC_INSTRUCTION_NEW; goto out; } - qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", tmp->path.constData(), fs->path.constData()); + qCDebug(lcUpdate, "remote rename detected based on fileid %s --> %s", base._path.constData(), fs->path.constData()); fs->instruction = CSYNC_INSTRUCTION_EVAL_RENAME; if (fs->type == CSYNC_FTW_TYPE_DIR) { - csync_rename_record(ctx, tmp->path, fs->path); + csync_rename_record(ctx, base._path, fs->path); } else { - if( tmp->etag != fs->etag ) { + if( base._etag != fs->etag ) { /* CSYNC_LOG(CSYNC_LOG_PRIORITY_DEBUG, "ETags are different!"); */ /* File with different etag, don't do a rename, but download the file again */ fs->instruction = CSYNC_INSTRUCTION_NEW; @@ -442,10 +433,59 @@ int csync_walker(CSYNC *ctx, std::unique_ptr<csync_file_stat_t> fs) { static bool fill_tree_from_db(CSYNC *ctx, const char *uri) { - if( csync_statedb_get_below_path(ctx, uri) < 0 ) { - qCWarning(lcUpdate, "StateDB could not be read!"); + int64_t count = 0; + QByteArray skipbase; + auto rowCallback = [ctx, &count, &skipbase](const OCC::SyncJournalFileRecord &rec) { + /* When selective sync is used, the database may have subtrees with a parent + * whose etag (md5) is _invalid_. These are ignored and shall not appear in the + * remote tree. + * Sometimes folders that are not ignored by selective sync get marked as + * _invalid_, but that is not a problem as the next discovery will retrieve + * their correct etags again and we don't run into this case. + */ + if( rec._etag == "_invalid_") { + qCDebug(lcUpdate, "%s selective sync excluded", rec._path.constData()); + skipbase = rec._path; + skipbase += '/'; + return; + } + + /* Skip over all entries with the same base path. Note that this depends + * strongly on the ordering of the retrieved items. */ + if( !skipbase.isEmpty() && rec._path.startsWith(skipbase) ) { + qCDebug(lcUpdate, "%s selective sync excluded because the parent is", rec._path.constData()); + return; + } else { + skipbase.clear(); + } + + std::unique_ptr<csync_file_stat_t> st = csync_file_stat_t::fromSyncJournalFileRecord(rec); + + /* Check for exclusion from the tree. + * Note that this is only a safety net in case the ignore list changes + * without a full remote discovery being triggered. */ + CSYNC_EXCLUDE_TYPE excluded = csync_excluded_traversal(ctx->excludes, st->path, st->type); + if (excluded != CSYNC_NOT_EXCLUDED) { + qDebug(lcUpdate, "%s excluded (%d)", st->path.constData(), excluded); + + if (excluded == CSYNC_FILE_EXCLUDE_AND_REMOVE + || excluded == CSYNC_FILE_SILENTLY_EXCLUDED) { + return; + } + + st->instruction = CSYNC_INSTRUCTION_IGNORE; + } + + /* store into result list. */ + ctx->remote.files[rec._path] = std::move(st); + ++count; + }; + + if (!ctx->statedb->getFilesBelowPath(uri, rowCallback)) { + ctx->status_code = CSYNC_STATUS_STATEDB_LOAD_ERROR; return false; } + qDebug(lcUpdate, "%" PRId64 " entries read below path %s from db.", count, uri); return true; } diff --git a/src/csync/csync_util.cpp b/src/csync/csync_util.cpp index d90a2eacc..afafafa5b 100644 --- a/src/csync/csync_util.cpp +++ b/src/csync/csync_util.cpp @@ -36,7 +36,6 @@ #define CSYNC_LOG_CATEGORY_NAME "csync.util" #include "csync_log.h" -#include "csync_statedb.h" typedef struct { const char *instr_str; diff --git a/src/csync/vio/csync_vio.cpp b/src/csync/vio/csync_vio.cpp index 611fb111b..3e9403c0e 100644 --- a/src/csync/vio/csync_vio.cpp +++ b/src/csync/vio/csync_vio.cpp @@ -30,7 +30,6 @@ #include "csync_util.h" #include "vio/csync_vio.h" #include "vio/csync_vio_local.h" -#include "csync_statedb.h" #include "common/c_jhash.h" csync_vio_handle_t *csync_vio_opendir(CSYNC *ctx, const char *name) { diff --git a/src/gui/folder.cpp b/src/gui/folder.cpp index 9f20f53fd..c61c4ea68 100644 --- a/src/gui/folder.cpp +++ b/src/gui/folder.cpp @@ -470,7 +470,7 @@ void Folder::slotWatchedPathChanged(const QString &path) SyncJournalFileRecord record; if (_journal.getFileRecord(relativePath, &record) && record.isValid() - && !FileSystem::fileChanged(path, record._fileSize, Utility::qDateTimeToTime_t(record._modtime))) { + && !FileSystem::fileChanged(path, record._fileSize, record._modtime)) { qCInfo(lcFolder) << "Ignoring spurious notification for file" << relativePath; return; // probably a spurious notification } diff --git a/src/libsync/owncloudpropagator.cpp b/src/libsync/owncloudpropagator.cpp index 8af558fa7..4673e9b34 100644 --- a/src/libsync/owncloudpropagator.cpp +++ b/src/libsync/owncloudpropagator.cpp @@ -112,13 +112,13 @@ PropagateItemJob::~PropagateItemJob() } } -static time_t getMinBlacklistTime() +static qint64 getMinBlacklistTime() { return qMax(qgetenv("OWNCLOUD_BLACKLIST_TIME_MIN").toInt(), 25); // 25 seconds } -static time_t getMaxBlacklistTime() +static qint64 getMaxBlacklistTime() { int v = qgetenv("OWNCLOUD_BLACKLIST_TIME_MAX").toInt(); if (v > 0) @@ -142,15 +142,15 @@ static SyncJournalErrorBlacklistRecord createBlacklistEntry( entry._renameTarget = item._renameTarget; entry._retryCount = old._retryCount + 1; - static time_t minBlacklistTime(getMinBlacklistTime()); - static time_t maxBlacklistTime(qMax(getMaxBlacklistTime(), minBlacklistTime)); + static qint64 minBlacklistTime(getMinBlacklistTime()); + static qint64 maxBlacklistTime(qMax(getMaxBlacklistTime(), minBlacklistTime)); // The factor of 5 feels natural: 25s, 2 min, 10 min, ~1h, ~5h, ~24h entry._ignoreDuration = old._ignoreDuration * 5; if (item._httpErrorCode == 403) { qCWarning(lcPropagator) << "Probably firewall error: " << item._httpErrorCode << ", blacklisting up to 1h only"; - entry._ignoreDuration = qMin(entry._ignoreDuration, time_t(60 * 60)); + entry._ignoreDuration = qMin(entry._ignoreDuration, qint64(60 * 60)); } else if (item._httpErrorCode == 413 || item._httpErrorCode == 415) { qCWarning(lcPropagator) << "Fatal Error condition" << item._httpErrorCode << ", maximum blacklist ignore time!"; diff --git a/src/libsync/propagateremotemove.cpp b/src/libsync/propagateremotemove.cpp index 74152f028..37ca9fecd 100644 --- a/src/libsync/propagateremotemove.cpp +++ b/src/libsync/propagateremotemove.cpp @@ -171,7 +171,7 @@ void PropagateRemoteMove::finalize() propagator()->_journal->deleteFileRecord(_item->_originalFile); SyncJournalFileRecord record = _item->toSyncJournalFileRecordWithInode(propagator()->getFilePath(_item->_renameTarget)); - record._path = _item->_renameTarget; + record._path = _item->_renameTarget.toUtf8(); if (oldRecord.isValid()) { record._checksumHeader = oldRecord._checksumHeader; if (record._fileSize != oldRecord._fileSize) { diff --git a/src/libsync/propagateuploadng.cpp b/src/libsync/propagateuploadng.cpp index 6d3ddfb80..5026f28a3 100644 --- a/src/libsync/propagateuploadng.cpp +++ b/src/libsync/propagateuploadng.cpp @@ -83,7 +83,7 @@ void PropagateUploadFileNG::doStartUpload() propagator()->_activeJobList.append(this); const SyncJournalDb::UploadInfo progressInfo = propagator()->_journal->getUploadInfo(_item->_file); - if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item->_modtime) { + if (progressInfo._valid && progressInfo._modtime == _item->_modtime) { _transferId = progressInfo._transferid; auto url = chunkUrl(); auto job = new LsColJob(propagator()->account(), url, this); @@ -229,7 +229,7 @@ void PropagateUploadFileNG::startNewUpload() SyncJournalDb::UploadInfo pi; pi._valid = true; pi._transferid = _transferId; - pi._modtime = Utility::qDateTimeFromTime_t(_item->_modtime); + pi._modtime = _item->_modtime; propagator()->_journal->setUploadInfo(_item->_file, pi); propagator()->_journal->commit("Upload info"); QMap<QByteArray, QByteArray> headers; diff --git a/src/libsync/propagateuploadv1.cpp b/src/libsync/propagateuploadv1.cpp index 0d0b2cd0f..c8b0e421d 100644 --- a/src/libsync/propagateuploadv1.cpp +++ b/src/libsync/propagateuploadv1.cpp @@ -43,7 +43,7 @@ void PropagateUploadFileV1::doStartUpload() const SyncJournalDb::UploadInfo progressInfo = propagator()->_journal->getUploadInfo(_item->_file); - if (progressInfo._valid && Utility::qDateTimeToTime_t(progressInfo._modtime) == _item->_modtime) { + if (progressInfo._valid && progressInfo._modtime == _item->_modtime) { _startChunk = progressInfo._chunk; _transferId = progressInfo._transferid; qCInfo(lcPropagateUpload) << _item->_file << ": Resuming from chunk " << _startChunk; @@ -272,7 +272,7 @@ void PropagateUploadFileV1::slotPutFinished() } pi._chunk = (currentChunk + _startChunk + 1) % _chunkCount; // next chunk to start with pi._transferid = _transferId; - pi._modtime = Utility::qDateTimeFromTime_t(_item->_modtime); + pi._modtime = _item->_modtime; pi._errorCount = 0; // successful chunk upload resets propagator()->_journal->setUploadInfo(_item->_file, pi); propagator()->_journal->commit("Upload info"); diff --git a/src/libsync/propagatorjobs.cpp b/src/libsync/propagatorjobs.cpp index 949f36b4b..49ecfe156 100644 --- a/src/libsync/propagatorjobs.cpp +++ b/src/libsync/propagatorjobs.cpp @@ -240,7 +240,7 @@ void PropagateLocalRename::start() _item->_file = _item->_renameTarget; SyncJournalFileRecord record = _item->toSyncJournalFileRecordWithInode(targetFile); - record._path = _item->_renameTarget; + record._path = _item->_renameTarget.toUtf8(); if (oldRecord.isValid()) { record._checksumHeader = oldRecord._checksumHeader; } diff --git a/src/libsync/syncengine.cpp b/src/libsync/syncengine.cpp index d34c516cd..f329d42cd 100644 --- a/src/libsync/syncengine.cpp +++ b/src/libsync/syncengine.cpp @@ -87,8 +87,7 @@ SyncEngine::SyncEngine(AccountPtr account, const QString &localPath, // Everything in the SyncEngine expects a trailing slash for the localPath. ASSERT(localPath.endsWith(QLatin1Char('/'))); - const QString dbFile = _journal->databaseFilePath(); - _csync_ctx.reset(new CSYNC(localPath.toUtf8().data(), dbFile.toUtf8().data())); + _csync_ctx.reset(new CSYNC(localPath.toUtf8().data(), journal)); _excludedFiles.reset(new ExcludedFiles(&_csync_ctx->excludes)); _syncFileStatusTracker.reset(new SyncFileStatusTracker(this)); diff --git a/src/libsync/syncfileitem.cpp b/src/libsync/syncfileitem.cpp index 0e6d8cb2a..bfea07163 100644 --- a/src/libsync/syncfileitem.cpp +++ b/src/libsync/syncfileitem.cpp @@ -26,8 +26,8 @@ Q_LOGGING_CATEGORY(lcFileItem, "sync.fileitem", QtInfoMsg) SyncJournalFileRecord SyncFileItem::toSyncJournalFileRecordWithInode(const QString &localFileName) { SyncJournalFileRecord rec; - rec._path = _file; - rec._modtime = Utility::qDateTimeFromTime_t(_modtime); + rec._path = _file.toUtf8(); + rec._modtime = _modtime; rec._type = _type; rec._etag = _etag; rec._fileId = _fileId; @@ -57,7 +57,7 @@ SyncFileItemPtr SyncFileItem::fromSyncJournalFileRecord(const SyncJournalFileRec SyncFileItemPtr item(new SyncFileItem); item->_file = rec._path; item->_inode = rec._inode; - item->_modtime = Utility::qDateTimeToTime_t(rec._modtime); + item->_modtime = rec._modtime; item->_type = static_cast<SyncFileItem::Type>(rec._type); item->_etag = rec._etag; item->_fileId = rec._fileId; diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index d6b68cb5c..d5cc8615f 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -33,7 +33,6 @@ IF( APPLE ) list(APPEND FolderWatcher_SRC ../src/gui/folderwatcher_mac.cpp) list(APPEND FolderWatcher_SRC ../src/gui/socketapisocket_mac.mm) ENDIF() -owncloud_add_test(CSyncSqlite "") owncloud_add_test(NetrcParser ../src/cmd/netrcparser.cpp) owncloud_add_test(OwnSql "") owncloud_add_test(SyncJournalDB "") diff --git a/test/csync/CMakeLists.txt b/test/csync/CMakeLists.txt index 237c52e27..5cb0590e4 100644 --- a/test/csync/CMakeLists.txt +++ b/test/csync/CMakeLists.txt @@ -32,13 +32,9 @@ add_cmocka_test(check_std_c_time std_tests/check_std_c_time.c ${TEST_TARGET_LIBR add_cmocka_test(check_csync_log csync_tests/check_csync_log.cpp ${TEST_TARGET_LIBRARIES}) add_cmocka_test(check_csync_exclude csync_tests/check_csync_exclude.cpp ${TEST_TARGET_LIBRARIES}) -add_cmocka_test(check_csync_statedb_load csync_tests/check_csync_statedb_load.cpp ${TEST_TARGET_LIBRARIES}) add_cmocka_test(check_csync_util csync_tests/check_csync_util.cpp ${TEST_TARGET_LIBRARIES}) add_cmocka_test(check_csync_misc csync_tests/check_csync_misc.cpp ${TEST_TARGET_LIBRARIES}) -# csync tests which require init -add_cmocka_test(check_csync_statedb_query csync_tests/check_csync_statedb_query.cpp ${TEST_TARGET_LIBRARIES}) - # vio add_cmocka_test(check_vio vio_tests/check_vio.cpp ${TEST_TARGET_LIBRARIES}) add_cmocka_test(check_vio_ext vio_tests/check_vio_ext.cpp ${TEST_TARGET_LIBRARIES}) diff --git a/test/csync/csync_tests/check_csync_exclude.cpp b/test/csync/csync_tests/check_csync_exclude.cpp index 0525942ac..3f0180e42 100644 --- a/test/csync/csync_tests/check_csync_exclude.cpp +++ b/test/csync/csync_tests/check_csync_exclude.cpp @@ -32,7 +32,7 @@ static int setup(void **state) { CSYNC *csync; - csync = new CSYNC("/tmp/check_csync1", ""); + csync = new CSYNC("/tmp/check_csync1", new OCC::SyncJournalDb("")); *state = csync; return 0; @@ -42,7 +42,7 @@ static int setup_init(void **state) { CSYNC *csync; int rc; - csync = new CSYNC("/tmp/check_csync1", ""); + csync = new CSYNC("/tmp/check_csync1", new OCC::SyncJournalDb("")); rc = csync_exclude_load(EXCLUDE_LIST_FILE, &(csync->excludes)); assert_int_equal(rc, 0); @@ -67,7 +67,9 @@ static int teardown(void **state) { CSYNC *csync = (CSYNC*)*state; int rc; + auto statedb = csync->statedb; delete csync; + delete statedb; rc = system("rm -rf /tmp/check_csync1"); assert_int_equal(rc, 0); diff --git a/test/csync/csync_tests/check_csync_statedb_load.cpp b/test/csync/csync_tests/check_csync_statedb_load.cpp deleted file mode 100644 index f5761f9ff..000000000 --- a/test/csync/csync_tests/check_csync_statedb_load.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * libcsync -- a library to sync a directory with another - * - * Copyright (c) 2008-2013 by Andreas Schneider <asn@cryptomilk.org> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ -#include <string.h> - -#define CSYNC_TEST 1 -#include "csync_statedb.cpp" - -#include "torture.h" - -#define TESTDB "/tmp/check_csync1/test.db" - -static int setup(void **state) { - CSYNC *csync; - int rc; - - rc = system("rm -rf /tmp/check_csync1"); - assert_int_equal(rc, 0); - - rc = system("mkdir -p /tmp/check_csync1"); - assert_int_equal(rc, 0); - - csync = new CSYNC("/tmp/check_csync1", TESTDB); - *state = csync; - - sqlite3 *db = NULL; - rc = sqlite3_open_v2(TESTDB, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, NULL); - assert_int_equal(rc, SQLITE_OK); - - rc = sqlite3_close(db); - assert_int_equal(rc, SQLITE_OK); - - return 0; -} - -static int teardown(void **state) { - CSYNC *csync = (CSYNC*)*state; - int rc; - - delete csync; - - rc = system("rm -rf /tmp/check_csync1"); - assert_int_equal(rc, 0); - - *state = NULL; - - return 0; -} - -static void check_csync_statedb_load(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - int rc; - - rc = csync_statedb_load(csync, TESTDB, &csync->statedb.db); - assert_int_equal(rc, 0); - - sqlite3_close(csync->statedb.db); -} - -static void check_csync_statedb_close(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - csync_stat_t sb; - time_t modtime; - mbchar_t *testdb = c_utf8_path_to_locale(TESTDB); - int rc; - - /* statedb not written */ - csync_statedb_load(csync, TESTDB, &csync->statedb.db); - - rc = _tstat(testdb, &sb); - assert_int_equal(rc, 0); - modtime = sb.st_mtime; - - rc = csync_statedb_close(csync); - assert_int_equal(rc, 0); - - rc = _tstat(testdb, &sb); - assert_int_equal(rc, 0); - assert_int_equal(modtime, sb.st_mtime); - - csync_statedb_load(csync, TESTDB, &csync->statedb.db); - - rc = _tstat(testdb, &sb); - assert_int_equal(rc, 0); - modtime = sb.st_mtime; - - /* wait a sec or the modtime will be the same */ - sleep(1); - - /* statedb written */ - rc = csync_statedb_close(csync); - assert_int_equal(rc, 0); - - rc = _tstat(testdb, &sb); - assert_int_equal(rc, 0); - - c_free_locale_string(testdb); -} - -int torture_run_tests(void) -{ - const struct CMUnitTest tests[] = { - cmocka_unit_test_setup_teardown(check_csync_statedb_load, setup, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_close, setup, teardown), - }; - - return cmocka_run_group_tests(tests, NULL, NULL); -} - diff --git a/test/csync/csync_tests/check_csync_statedb_query.cpp b/test/csync/csync_tests/check_csync_statedb_query.cpp deleted file mode 100644 index 7f3988a71..000000000 --- a/test/csync/csync_tests/check_csync_statedb_query.cpp +++ /dev/null @@ -1,217 +0,0 @@ -/* - * libcsync -- a library to sync a directory with another - * - * Copyright (c) 2008-2013 by Andreas Schneider <asn@cryptomilk.org> - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; either - * version 2.1 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA - */ - -#define CSYNC_TEST 1 -#include "csync_statedb.cpp" - -#include "torture.h" - -#define TESTDB "/tmp/check_csync1/test.db" -#define TESTDBTMP "/tmp/check_csync1/test.db.ctmp" - - -static int setup(void **state) -{ - CSYNC *csync; - int rc = 0; - - rc = system("rm -rf /tmp/check_csync1"); - assert_int_equal(rc, 0); - rc = system("mkdir -p /tmp/check_csync1"); - assert_int_equal(rc, 0); - rc = system("mkdir -p /tmp/check_csync"); - assert_int_equal(rc, 0); - csync = new CSYNC("/tmp/check_csync1", TESTDB); - - sqlite3 *db = NULL; - rc = sqlite3_open_v2(TESTDB, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, NULL); - assert_int_equal(rc, SQLITE_OK); - rc = sqlite3_close(db); - assert_int_equal(rc, SQLITE_OK); - - rc = csync_statedb_load(csync, TESTDB, &csync->statedb.db); - assert_int_equal(rc, 0); - - *state = csync; - - return 0; -} - -static int setup_db(void **state) -{ - char *errmsg; - int rc = 0; - sqlite3 *db = NULL; - - const char *sql = "CREATE TABLE IF NOT EXISTS metadata (" - "phash INTEGER(8)," - "pathlen INTEGER," - "path VARCHAR(4096)," - "inode INTEGER," - "uid INTEGER," - "gid INTEGER," - "mode INTEGER," - "modtime INTEGER(8)," - "type INTEGER," - "md5 VARCHAR(32)," - "PRIMARY KEY(phash)" - ");"; - - const char *sql2 = "INSERT INTO metadata" - "(phash, pathlen, path, inode, uid, gid, mode, modtime, type, md5) VALUES" - "(42, 42, 'Its funny stuff', 23, 42, 43, 55, 66, 2, 54);"; - - - setup(state); - rc = sqlite3_open( TESTDB, &db); - assert_int_equal(rc, SQLITE_OK); - - rc = sqlite3_exec( db, sql, NULL, NULL, &errmsg ); - assert_int_equal(rc, SQLITE_OK); - - rc = sqlite3_exec( db, sql2, NULL, NULL, &errmsg ); - assert_int_equal(rc, SQLITE_OK); - - sqlite3_close(db); - - return 0; - -} - -static int teardown(void **state) { - CSYNC *csync = (CSYNC*)*state; - int rc = 0; - - delete csync; - rc = system("rm -rf /tmp/check_csync"); - assert_int_equal(rc, 0); - rc = system("rm -rf /tmp/check_csync1"); - assert_int_equal(rc, 0); - - *state = NULL; - - return 0; -} - - -static void check_csync_statedb_query_statement(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - c_strlist_t *result; - - result = csync_statedb_query(csync->statedb.db, ""); - assert_null(result); - if (result != NULL) { - c_strlist_destroy(result); - } - - result = csync_statedb_query(csync->statedb.db, "SELECT;"); - assert_null(result); - if (result != NULL) { - c_strlist_destroy(result); - } -} - -static void check_csync_statedb_drop_tables(void **state) -{ - // CSYNC *csync = (CSYNC*)*state; - int rc = 0; - (void) state; - - // rc = csync_statedb_drop_tables(csync->statedb.db); - assert_int_equal(rc, 0); - // rc = csync_statedb_create_tables(csync->statedb.db); - assert_int_equal(rc, 0); - // rc = csync_statedb_drop_tables(csync->statedb.db); - assert_int_equal(rc, 0); -} - -static void check_csync_statedb_insert_metadata(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - std::unique_ptr<csync_file_stat_t> st; - int i, rc = 0; - - // rc = csync_statedb_create_tables(csync->statedb.db); - assert_int_equal(rc, 0); - - for (i = 0; i < 100; i++) { - st.reset(new csync_file_stat_t); - st->path = QString("file_%1").arg(i).toUtf8(); - - csync->local.files[st->path] = std::move(st); - } - - // rc = csync_statedb_insert_metadata(csync, csync->statedb.db); - assert_int_equal(rc, 0); -} - -static void check_csync_statedb_write(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - std::unique_ptr<csync_file_stat_t> st; - int i, rc = 0; - - for (i = 0; i < 100; i++) { - st.reset(new csync_file_stat_t); - st->path = QString("file_%1").arg(i).toUtf8(); - - csync->local.files[st->path] = std::move(st); - assert_int_equal(rc, 0); - } - - // rc = csync_statedb_write(csync, csync->statedb.db); - assert_int_equal(rc, 0); -} - - -static void check_csync_statedb_get_stat_by_path_not_found(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - std::unique_ptr<csync_file_stat_t> tmp; - - tmp = csync_statedb_get_stat_by_path(csync, "666"); - assert_null(tmp.get()); -} - - -static void check_csync_statedb_get_stat_by_inode_not_found(void **state) -{ - CSYNC *csync = (CSYNC*)*state; - std::unique_ptr<csync_file_stat_t> tmp; - - tmp = csync_statedb_get_stat_by_inode(csync, (ino_t) 666); - assert_null(tmp.get()); -} - -int torture_run_tests(void) -{ - const struct CMUnitTest tests[] = { - cmocka_unit_test_setup_teardown(check_csync_statedb_query_statement, setup, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_drop_tables, setup, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_insert_metadata, setup, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_write, setup, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_path_not_found, setup_db, teardown), - cmocka_unit_test_setup_teardown(check_csync_statedb_get_stat_by_inode_not_found, setup_db, teardown), - }; - - return cmocka_run_group_tests(tests, NULL, NULL); -} - diff --git a/test/csync/csync_tests/check_csync_update.cpp b/test/csync/csync_tests/check_csync_update.cpp index 4fae4e1a7..db6819907 100644 --- a/test/csync/csync_tests/check_csync_update.cpp +++ b/test/csync/csync_tests/check_csync_update.cpp @@ -98,12 +98,10 @@ static int setup(void **state) assert_int_equal(rc, 0); rc = system("mkdir -p /tmp/check_csync1"); assert_int_equal(rc, 0); - csync = new CSYNC("/tmp/check_csync1", TESTDB); /* Create a new db with metadata */ sqlite3 *db; - // csync->statedb.file = c_strdup(TESTDB); - rc = sqlite3_open(csync->statedb.file, &db); + rc = sqlite3_open(TESTDB, &db); statedb_create_metadata_table(db); if( firstrun ) { statedb_insert_metadata(db); @@ -111,8 +109,8 @@ static int setup(void **state) } sqlite3_close(db); - rc = csync_statedb_load(csync, TESTDB, &csync->statedb.db); - assert_int_equal(rc, 0); + csync = new CSYNC("/tmp/check_csync1", new OCC::SyncJournalDb(TESTDB)); + assert_true(csync->statedb->isConnected()); *state = csync; @@ -128,7 +126,7 @@ static int setup_ftw(void **state) assert_int_equal(rc, 0); rc = system("mkdir -p /tmp/check_csync1"); assert_int_equal(rc, 0); - csync = new CSYNC("/tmp", TESTDB); + csync = new CSYNC("/tmp", new OCC::SyncJournalDb(TESTDB)); sqlite3 *db = NULL; rc = sqlite3_open_v2(TESTDB, &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, NULL); @@ -137,10 +135,9 @@ static int setup_ftw(void **state) rc = sqlite3_close(db); assert_int_equal(rc, SQLITE_OK); - rc = csync_statedb_load(csync, TESTDB, &csync->statedb.db); - assert_int_equal(rc, 0); + csync = new CSYNC("/tmp", new OCC::SyncJournalDb(TESTDB)); + assert_true(csync->statedb->isConnected()); - csync->statedb.file = c_strdup( TESTDB ); *state = csync; return 0; @@ -150,8 +147,10 @@ static int teardown(void **state) { CSYNC *csync = (CSYNC*)*state; - unlink( csync->statedb.file); + unlink(TESTDB); + auto statedb = csync->statedb; delete csync; + delete statedb; *state = NULL; @@ -288,7 +287,6 @@ static void check_csync_detect_update_db_rename(void **state) int rc = 0; fs = create_fstat("wurst.txt", 0, 42); - csync_set_statedb_exists(csync, 1); rc = _csync_detect_update(csync, std::move(fs)); assert_int_equal(rc, 0); diff --git a/test/csync/vio_tests/check_vio.cpp b/test/csync/vio_tests/check_vio.cpp index 0607dcaa8..884accf43 100644 --- a/test/csync/vio_tests/check_vio.cpp +++ b/test/csync/vio_tests/check_vio.cpp @@ -49,7 +49,7 @@ static int setup(void **state) rc = system("rm -rf /tmp/csync_test"); assert_int_equal(rc, 0); - csync = new CSYNC("/tmp/check_csync1", ""); + csync = new CSYNC("/tmp/check_csync1", new OCC::SyncJournalDb("")); csync->current = LOCAL_REPLICA; @@ -78,7 +78,9 @@ static int teardown(void **state) { CSYNC *csync = (CSYNC*)*state; int rc; + auto statedb = csync->statedb; delete csync; + delete statedb; rc = chdir(wd_buffer); assert_int_equal(rc, 0); diff --git a/test/csync/vio_tests/check_vio_ext.cpp b/test/csync/vio_tests/check_vio_ext.cpp index 625285ce0..f86ce9591 100644 --- a/test/csync/vio_tests/check_vio_ext.cpp +++ b/test/csync/vio_tests/check_vio_ext.cpp @@ -97,7 +97,7 @@ static int setup_testenv(void **state) { statevar *mystate = (statevar*)malloc( sizeof(statevar) ); mystate->result = NULL; - mystate->csync = new CSYNC("/tmp/check_csync1", ""); + mystate->csync = new CSYNC("/tmp/check_csync1", new OCC::SyncJournalDb("")); mystate->csync->current = LOCAL_REPLICA; @@ -124,7 +124,9 @@ static int teardown(void **state) { output("================== Tearing down!\n"); + auto statedb = csync->statedb; delete csync; + delete statedb; rc = _tchdir(wd_buffer); assert_int_equal(rc, 0); diff --git a/test/testcsyncsqlite.cpp b/test/testcsyncsqlite.cpp deleted file mode 100644 index 093ee43cb..000000000 --- a/test/testcsyncsqlite.cpp +++ /dev/null @@ -1,80 +0,0 @@ -/* - This software is in the public domain, furnished "as is", without technical - support, and with no warranty, express or implied, as to its usefulness for - any purpose. -*/ - -#include "csync_statedb.h" -#include "csync_private.h" -#include <QtTest> - - -class TestCSyncSqlite : public QObject -{ - Q_OBJECT - -private: - CSYNC *_ctx; -private slots: - void initTestCase() { - int rc; - - QString db = QCoreApplication::applicationDirPath() + "/test_journal.db"; - _ctx = new CSYNC("/tmp/check_csync1", db.toLocal8Bit()); - - rc = csync_statedb_load(_ctx, _ctx->statedb.file, &(_ctx->statedb.db)); - QVERIFY(rc == 0); - } - - void testFullResult() { - std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_path( _ctx, "test2/zu/zuzu" ); - QVERIFY(st.get()); - QCOMPARE( QString::fromUtf8(st->path), QLatin1String("test2/zu/zuzu") ); - QCOMPARE( QString::number(st->inode), QString::number(1709554)); - QCOMPARE( QString::number(st->modtime), QString::number(1384415006)); - QCOMPARE( QString::number(st->type), QString::number(2)); - QCOMPARE( QString::fromUtf8(st->etag), QLatin1String("52847f2090665")); - QCOMPARE( QString::fromUtf8(st->file_id), QLatin1String("00000557525d5af3d9625")); - - } - - void testByHash() { - std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_path(_ctx, "documents/c1"); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1")); - - st = csync_statedb_get_stat_by_path(_ctx, "documents/c1/c2"); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("documents/c1/c2")); - } - - void testByInode() { - std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_inode(_ctx, 1709555); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("test2/zu/zuzu/zuzuzu")); - - st = csync_statedb_get_stat_by_inode(_ctx, 1706571); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("Shared/for_kf/a2")); - } - - void testByFileId() { - std::unique_ptr<csync_file_stat_t> st = csync_statedb_get_stat_by_file_id(_ctx, "00000556525d5af3d9625"); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("test2/zu")); - - st = csync_statedb_get_stat_by_file_id(_ctx, "-0000001525d5af3d9625"); - QVERIFY(st.get()); - QCOMPARE(QString::fromUtf8(st->path), QLatin1String("Shared")); - } - - void cleanupTestCase() { - csync_statedb_close(_ctx); - delete _ctx; - _ctx = nullptr; - } - -}; - -QTEST_GUILESS_MAIN(TestCSyncSqlite) -#include "testcsyncsqlite.moc" diff --git a/test/testsyncengine.cpp b/test/testsyncengine.cpp index 032937b62..a4c4c5be4 100644 --- a/test/testsyncengine.cpp +++ b/test/testsyncengine.cpp @@ -183,7 +183,7 @@ private slots: // Remove subFolderA with selectiveSync: fakeFolder.syncEngine().journal()->setSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, {"parentFolder/subFolderA/"}); - fakeFolder.syncEngine().journal()->avoidReadFromDbOnNextSync("parentFolder/subFolderA/"); + fakeFolder.syncEngine().journal()->avoidReadFromDbOnNextSync(QByteArrayLiteral("parentFolder/subFolderA/")); fakeFolder.syncOnce(); @@ -238,7 +238,7 @@ private slots: // folders are uploaded anyway is some circumstances. FakeFolder fakeFolder{FileInfo{ QString(), { FileInfo { QStringLiteral("parentFolder"), { - FileInfo{ QStringLiteral("subFolder"), { + FileInfo{ QStringLiteral("subFolderA"), { { QStringLiteral("fileA.txt"), 400 }, { QStringLiteral("fileB.txt"), 400, 'o' }, FileInfo { QStringLiteral("subsubFolder"), { @@ -252,23 +252,24 @@ private slots: { QStringLiteral("fileF.txt"), 400, 'o' } }} }} - }} + }}, + FileInfo{ QStringLiteral("subFolderB"), {} } }} }}}; QCOMPARE(fakeFolder.currentLocalState(), fakeFolder.currentRemoteState()); auto expectedServerState = fakeFolder.currentRemoteState(); - // Remove subFolder with selectiveSync: + // Remove subFolderA with selectiveSync: fakeFolder.syncEngine().journal()->setSelectiveSyncList(SyncJournalDb::SelectiveSyncBlackList, - {"parentFolder/subFolder/"}); - fakeFolder.syncEngine().journal()->avoidReadFromDbOnNextSync("parentFolder/subFolder/"); + {"parentFolder/subFolderA/"}); + fakeFolder.syncEngine().journal()->avoidReadFromDbOnNextSync(QByteArrayLiteral("parentFolder/subFolderA/")); // But touch local file before the next sync, such that the local folder // can't be removed - fakeFolder.localModifier().setContents("parentFolder/subFolder/fileB.txt", 'n'); - fakeFolder.localModifier().setContents("parentFolder/subFolder/subsubFolder/fileD.txt", 'n'); - fakeFolder.localModifier().setContents("parentFolder/subFolder/anotherFolder/subsubFolder/fileF.txt", 'n'); + fakeFolder.localModifier().setContents("parentFolder/subFolderA/fileB.txt", 'n'); + fakeFolder.localModifier().setContents("parentFolder/subFolderA/subsubFolder/fileD.txt", 'n'); + fakeFolder.localModifier().setContents("parentFolder/subFolderA/anotherFolder/subsubFolder/fileF.txt", 'n'); // Several follow-up syncs don't change the state at all, // in particular the remote state doesn't change and fileB.txt @@ -282,14 +283,15 @@ private slots: QCOMPARE(fakeFolder.currentRemoteState(), expectedServerState); // The local state should still have subFolderA auto local = fakeFolder.currentLocalState(); - QVERIFY(local.find("parentFolder/subFolder")); - QVERIFY(!local.find("parentFolder/subFolder/fileA.txt")); - QVERIFY(local.find("parentFolder/subFolder/fileB.txt")); - QVERIFY(!local.find("parentFolder/subFolder/subsubFolder/fileC.txt")); - QVERIFY(local.find("parentFolder/subFolder/subsubFolder/fileD.txt")); - QVERIFY(!local.find("parentFolder/subFolder/anotherFolder/subsubFolder/fileE.txt")); - QVERIFY(local.find("parentFolder/subFolder/anotherFolder/subsubFolder/fileF.txt")); - QVERIFY(!local.find("parentFolder/subFolder/anotherFolder/emptyFolder")); + QVERIFY(local.find("parentFolder/subFolderA")); + QVERIFY(!local.find("parentFolder/subFolderA/fileA.txt")); + QVERIFY(local.find("parentFolder/subFolderA/fileB.txt")); + QVERIFY(!local.find("parentFolder/subFolderA/subsubFolder/fileC.txt")); + QVERIFY(local.find("parentFolder/subFolderA/subsubFolder/fileD.txt")); + QVERIFY(!local.find("parentFolder/subFolderA/anotherFolder/subsubFolder/fileE.txt")); + QVERIFY(local.find("parentFolder/subFolderA/anotherFolder/subsubFolder/fileF.txt")); + QVERIFY(!local.find("parentFolder/subFolderA/anotherFolder/emptyFolder")); + QVERIFY(local.find("parentFolder/subFolderB")); } } } diff --git a/test/testsyncjournaldb.cpp b/test/testsyncjournaldb.cpp index 5219dd8cf..0575c0ed5 100644 --- a/test/testsyncjournaldb.cpp +++ b/test/testsyncjournaldb.cpp @@ -26,9 +26,9 @@ public: QVERIFY(_tempDir.isValid()); } - QDateTime dropMsecs(QDateTime time) + qint64 dropMsecs(QDateTime time) { - return Utility::qDateTimeFromTime_t(Utility::qDateTimeToTime_t(time)); + return Utility::qDateTimeToTime_t(time); } private slots: @@ -46,7 +46,7 @@ private slots: void testFileRecord() { SyncJournalFileRecord record; - QVERIFY(_db.getFileRecord("nonexistant", &record)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("nonexistant"), &record)); QVERIFY(!record.isValid()); record._path = "foo"; @@ -61,13 +61,13 @@ private slots: QVERIFY(_db.setFileRecord(record)); SyncJournalFileRecord storedRecord; - QVERIFY(_db.getFileRecord("foo", &storedRecord)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo"), &storedRecord)); QVERIFY(storedRecord == record); // Update checksum record._checksumHeader = "Adler32:newchecksum"; _db.updateFileRecordChecksum("foo", "newchecksum", "Adler32"); - QVERIFY(_db.getFileRecord("foo", &storedRecord)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo"), &storedRecord)); QVERIFY(storedRecord == record); // Update metadata @@ -79,11 +79,11 @@ private slots: record._remotePerm = RemotePermissions("NV"); record._fileSize = 289055; _db.setFileRecordMetadata(record); - QVERIFY(_db.getFileRecord("foo", &storedRecord)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo"), &storedRecord)); QVERIFY(storedRecord == record); QVERIFY(_db.deleteFileRecord("foo")); - QVERIFY(_db.getFileRecord("foo", &record)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo"), &record)); QVERIFY(!record.isValid()); } @@ -95,11 +95,11 @@ private slots: record._path = "foo-checksum"; record._remotePerm = RemotePermissions("RW"); record._checksumHeader = "MD5:mychecksum"; - record._modtime = QDateTime::currentDateTimeUtc(); + record._modtime = Utility::qDateTimeToTime_t(QDateTime::currentDateTimeUtc()); QVERIFY(_db.setFileRecord(record)); SyncJournalFileRecord storedRecord; - QVERIFY(_db.getFileRecord("foo-checksum", &storedRecord)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo-checksum"), &storedRecord)); QVERIFY(storedRecord._path == record._path); QVERIFY(storedRecord._remotePerm == record._remotePerm); QVERIFY(storedRecord._checksumHeader == record._checksumHeader); @@ -108,19 +108,19 @@ private slots: // Attention: compare time_t types here, as QDateTime seem to maintain // milliseconds internally, which disappear in sqlite. Go for full seconds here. - QVERIFY(storedRecord._modtime.toTime_t() == record._modtime.toTime_t()); + QVERIFY(storedRecord._modtime == record._modtime); QVERIFY(storedRecord == record); } { SyncJournalFileRecord record; record._path = "foo-nochecksum"; record._remotePerm = RemotePermissions("RWN"); - record._modtime = QDateTime::currentDateTimeUtc(); + record._modtime = Utility::qDateTimeToTime_t(QDateTime::currentDateTimeUtc()); QVERIFY(_db.setFileRecord(record)); SyncJournalFileRecord storedRecord; - QVERIFY(_db.getFileRecord("foo-nochecksum", &storedRecord)); + QVERIFY(_db.getFileRecord(QByteArrayLiteral("foo-nochecksum"), &storedRecord)); QVERIFY(storedRecord == record); } } diff --git a/test/testuploadreset.cpp b/test/testuploadreset.cpp index 9c92d9a1d..ba5489c3a 100644 --- a/test/testuploadreset.cpp +++ b/test/testuploadreset.cpp @@ -35,7 +35,7 @@ private slots: SyncJournalDb::UploadInfo uploadInfo; uploadInfo._transferid = 1; uploadInfo._valid = true; - uploadInfo._modtime = modTime; + uploadInfo._modtime = Utility::qDateTimeToTime_t(modTime); fakeFolder.syncEngine().journal()->setUploadInfo("A/a0", uploadInfo); fakeFolder.uploadState().mkdir("1"); |