1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
|
/*
* Copyright (C) by Olivier Goffart <ogoffart@woboq.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
* or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* for more details.
*/
#pragma once
#include <QObject>
#include <QElapsedTimer>
#include <QStringList>
#include <csync.h>
#include <QMap>
#include "networkjobs.h"
#include <QMutex>
#include <QWaitCondition>
#include <QLinkedList>
namespace OCC {
class Account;
/**
* The Discovery Phase was once called "update" phase in csync terms.
* Its goal is to look at the files in one of the remote and check compared to the db
* if the files are new, or changed.
*/
struct SyncOptions
{
SyncOptions()
: _newBigFolderSizeLimit(-1)
, _confirmExternalStorage(false)
, _initialChunkSize(10 * 1000 * 1000) // 10 MB
, _minChunkSize(1 * 1000 * 1000) // 1 MB
, _maxChunkSize(100 * 1000 * 1000) // 100 MB
, _targetChunkUploadDuration(60 * 1000) // 1 minute
{
}
/** Maximum size (in Bytes) a folder can have without asking for confirmation.
* -1 means infinite */
qint64 _newBigFolderSizeLimit;
/** If a confirmation should be asked for external storages */
bool _confirmExternalStorage;
/** The initial un-adjusted chunk size in bytes for chunked uploads, both
* for old and new chunking algorithm, which classifies the item to be chunked
*
* In chunkingNG, when dynamic chunk size adjustments are done, this is the
* starting value and is then gradually adjusted within the
* minChunkSize / maxChunkSize bounds.
*/
quint64 _initialChunkSize;
/** The minimum chunk size in bytes for chunked uploads */
quint64 _minChunkSize;
/** The maximum chunk size in bytes for chunked uploads */
quint64 _maxChunkSize;
/** The target duration of chunk uploads for dynamic chunk sizing.
*
* Set to 0 it will disable dynamic chunk sizing.
*/
quint64 _targetChunkUploadDuration;
};
/**
* @brief The FileStatPointer class
* @ingroup libsync
*/
class FileStatPointer
{
public:
FileStatPointer(csync_vio_file_stat_t *stat)
: _stat(stat)
{
}
FileStatPointer(const FileStatPointer &other)
: _stat(csync_vio_file_stat_copy(other._stat))
{
}
~FileStatPointer()
{
csync_vio_file_stat_destroy(_stat);
}
FileStatPointer &operator=(const FileStatPointer &other)
{
csync_vio_file_stat_destroy(_stat);
_stat = csync_vio_file_stat_copy(other._stat);
return *this;
}
inline csync_vio_file_stat_t *data() const { return _stat; }
inline csync_vio_file_stat_t *operator->() const { return _stat; }
private:
csync_vio_file_stat_t *_stat;
};
struct DiscoveryDirectoryResult
{
QString path;
QString msg;
int code;
QList<FileStatPointer> list;
int listIndex;
DiscoveryDirectoryResult()
: code(EIO)
, listIndex(0)
{
}
};
/**
* @brief The DiscoverySingleDirectoryJob class
*
* Run in the main thread, reporting to the DiscoveryJobMainThread object
*
* @ingroup libsync
*/
class DiscoverySingleDirectoryJob : public QObject
{
Q_OBJECT
public:
explicit DiscoverySingleDirectoryJob(const AccountPtr &account, const QString &path, QObject *parent = 0);
// Specify thgat this is the root and we need to check the data-fingerprint
void setIsRootPath() { _isRootPath = true; }
void start();
void abort();
// This is not actually a network job, it is just a job
signals:
void firstDirectoryPermissions(const QString &);
void etagConcatenation(const QString &);
void etag(const QString &);
void finishedWithResult(const QList<FileStatPointer> &);
void finishedWithError(int csyncErrnoCode, const QString &msg);
private slots:
void directoryListingIteratedSlot(QString, const QMap<QString, QString> &);
void lsJobFinishedWithoutErrorSlot();
void lsJobFinishedWithErrorSlot(QNetworkReply *);
private:
QList<FileStatPointer> _results;
QString _subPath;
QString _etagConcatenation;
QString _firstEtag;
AccountPtr _account;
// The first result is for the directory itself and need to be ignored.
// This flag is true if it was already ignored.
bool _ignoredFirst;
// Set to true if this is the root path and we need to check the data-fingerprint
bool _isRootPath;
// If this directory is an external storage (The first item has 'M' in its permission)
bool _isExternalStorage;
QPointer<LsColJob> _lsColJob;
public:
QByteArray _dataFingerprint;
};
// Lives in main thread. Deleted by the SyncEngine
class DiscoveryJob;
class DiscoveryMainThread : public QObject
{
Q_OBJECT
QPointer<DiscoveryJob> _discoveryJob;
QPointer<DiscoverySingleDirectoryJob> _singleDirJob;
QString _pathPrefix; // remote path
AccountPtr _account;
DiscoveryDirectoryResult *_currentDiscoveryDirectoryResult;
qint64 *_currentGetSizeResult;
bool _firstFolderProcessed;
public:
DiscoveryMainThread(AccountPtr account)
: QObject()
, _account(account)
, _currentDiscoveryDirectoryResult(0)
, _currentGetSizeResult(0)
, _firstFolderProcessed(false)
{
}
void abort();
QByteArray _dataFingerprint;
public slots:
// From DiscoveryJob:
void doOpendirSlot(const QString &url, DiscoveryDirectoryResult *);
void doGetSizeSlot(const QString &path, qint64 *result);
// From Job:
void singleDirectoryJobResultSlot(const QList<FileStatPointer> &);
void singleDirectoryJobFinishedWithErrorSlot(int csyncErrnoCode, const QString &msg);
void singleDirectoryJobFirstDirectoryPermissionsSlot(const QString &);
void slotGetSizeFinishedWithError();
void slotGetSizeResult(const QVariantMap &);
signals:
void etag(const QString &);
void etagConcatenation(const QString &);
public:
void setupHooks(DiscoveryJob *discoveryJob, const QString &pathPrefix);
};
/**
* @brief The DiscoveryJob class
*
* Lives in the other thread, deletes itself in !start()
*
* @ingroup libsync
*/
class DiscoveryJob : public QObject
{
Q_OBJECT
friend class DiscoveryMainThread;
CSYNC *_csync_ctx;
csync_log_callback _log_callback;
int _log_level;
QElapsedTimer _lastUpdateProgressCallbackCall;
/**
* return true if the given path should be ignored,
* false if the path should be synced
*/
bool isInSelectiveSyncBlackList(const char *path) const;
static int isInSelectiveSyncBlackListCallback(void *, const char *);
bool checkSelectiveSyncNewFolder(const QString &path, const char *remotePerm);
static int checkSelectiveSyncNewFolderCallback(void *data, const char *path, const char *remotePerm);
// Just for progress
static void update_job_update_callback(bool local,
const char *dirname,
void *userdata);
// For using QNAM to get the directory listings
static csync_vio_handle_t *remote_vio_opendir_hook(const char *url,
void *userdata);
static csync_vio_file_stat_t *remote_vio_readdir_hook(csync_vio_handle_t *dhandle,
void *userdata);
static void remote_vio_closedir_hook(csync_vio_handle_t *dhandle,
void *userdata);
QMutex _vioMutex;
QWaitCondition _vioWaitCondition;
public:
explicit DiscoveryJob(CSYNC *ctx, QObject *parent = 0)
: QObject(parent)
, _csync_ctx(ctx)
{
// We need to forward the log property as csync uses thread local
// and updates run in another thread
_log_callback = csync_get_log_callback();
_log_level = csync_get_log_level();
}
QStringList _selectiveSyncBlackList;
QStringList _selectiveSyncWhiteList;
SyncOptions _syncOptions;
Q_INVOKABLE void start();
signals:
void finished(int result);
void folderDiscovered(bool local, QString folderUrl);
// After the discovery job has been woken up again (_vioWaitCondition)
void doOpendirSignal(QString url, DiscoveryDirectoryResult *);
void doGetSizeSignal(const QString &path, qint64 *result);
// A new folder was discovered and was not synced because of the confirmation feature
void newBigFolder(const QString &folder, bool isExternal);
};
}
|