From 4e45cab2c1631f9202f0ac19d53c002b8d4cd90e Mon Sep 17 00:00:00 2001 From: Duncan Mac-Vicar P Date: Mon, 4 Apr 2011 18:41:14 +0200 Subject: - lot of stability improvements - the crash with threads goes away by using QMap - add a polling timer --- src/mirall/folder.h | 3 +- src/mirall/folderwatcher.cpp | 91 +++++++++++++++++++++++++++++++++++--------- src/mirall/folderwatcher.h | 31 +++++++++++++++ src/mirall/inotify.cpp | 36 ++++++++++++++---- src/mirall/inotify.h | 8 +++- src/mirall/syncresult.h | 21 ++++++++++ src/mirall/unisonfolder.cpp | 29 ++++++++++++-- src/mirall/unisonfolder.h | 3 ++ 8 files changed, 190 insertions(+), 32 deletions(-) create mode 100644 src/mirall/syncresult.h (limited to 'src') diff --git a/src/mirall/folder.h b/src/mirall/folder.h index 389f7d9a1..801281dad 100644 --- a/src/mirall/folder.h +++ b/src/mirall/folder.h @@ -44,10 +44,9 @@ signals: void syncFinished(); protected: - + FolderWatcher *_watcher; private: QString _path; - FolderWatcher *_watcher; QAction *_openAction; protected slots: diff --git a/src/mirall/folderwatcher.cpp b/src/mirall/folderwatcher.cpp index b682f9f87..483348e48 100644 --- a/src/mirall/folderwatcher.cpp +++ b/src/mirall/folderwatcher.cpp @@ -20,25 +20,38 @@ static const uint32_t standard_event_mask = /* minimum amount of seconds between two events to consider it a new event */ #define DEFAULT_EVENT_INTERVAL_SEC 5 +#define DEFAULT_POLL_INTERVAL_SEC 30 namespace Mirall { - FolderWatcher::FolderWatcher(const QString &root, QObject *parent) : QObject(parent), _eventsEnabled(true), _eventInterval(DEFAULT_EVENT_INTERVAL_SEC), + _pollInterval(DEFAULT_POLL_INTERVAL_SEC), _root(root), _processTimer(new QTimer(this)), + _pollTimer(new QTimer(this)), _lastMask(0) { + // this is not the best place for this + addIgnore("/**/.unison*"); + _processTimer->setSingleShot(true); QObject::connect(_processTimer, SIGNAL(timeout()), this, SLOT(slotProcessTimerTimeout())); + _pollTimer->setSingleShot(false); + _pollTimer->setInterval(pollInterval() * 1000); + QObject::connect(_pollTimer, SIGNAL(timeout()), this, SLOT(slotPollTimerTimeout())); + _pollTimer->start(); + _inotify = new INotify(standard_event_mask); slotAddFolderRecursive(root); QObject::connect(_inotify, SIGNAL(notifyEvent(int, int, const QString &)), SLOT(slotINotifyEvent(int, int, const QString &))); + // do a first synchronization to get changes while + // the application was not running + setProcessTimer(); } FolderWatcher::~FolderWatcher() @@ -51,6 +64,11 @@ QString FolderWatcher::root() const return _root; } +void FolderWatcher::addIgnore(const QString &pattern) +{ + _ignores.append(pattern); +} + bool FolderWatcher::eventsEnabled() const { return _eventsEnabled; @@ -84,6 +102,16 @@ void FolderWatcher::setEventInterval(int seconds) _eventInterval = seconds; } +int FolderWatcher::pollInterval() const +{ + return _pollInterval; +} + +void FolderWatcher::setPollInterval(int seconds) +{ + _pollInterval = seconds; +} + QStringList FolderWatcher::folders() const { return _inotify->directories(); @@ -102,6 +130,16 @@ void FolderWatcher::slotAddFolderRecursive(const QString &path) if (folder.exists() && !watchedFolders.contains(folder.path())) { subdirs++; //qDebug() << "(+) Watcher:" << folder.path(); + // check that it does not match the ignore list + foreach (QString pattern, _ignores) { + QRegExp regexp(pattern); + regexp.setPatternSyntax(QRegExp::Wildcard); + if (regexp.exactMatch(folder.path())) { + qDebug() << "* Not adding" << folder.path(); + continue; + } + + } _inotify->addPath(folder.path()); } else @@ -137,14 +175,17 @@ void FolderWatcher::slotINotifyEvent(int mask, int cookie, const QString &path) if (mask & IN_CREATE) { //qDebug() << cookie << " CREATE: " << path; if (QFileInfo(path).isDir()) { + //setEventsEnabled(false); slotAddFolderRecursive(path); + //setEventsEnabled(true); } } else if (mask & IN_DELETE) { //qDebug() << cookie << " DELETE: " << path; - if (_inotify->directories().contains(path)); + if (_inotify->directories().contains(path) && + QFileInfo(path).isDir()); qDebug() << "(-) Watcher:" << path; - _inotify->removePath(path); + _inotify->removePath(path); } else if (mask & IN_CLOSE_WRITE) { //qDebug() << cookie << " WRITABLE CLOSED: " << path; @@ -156,6 +197,15 @@ void FolderWatcher::slotINotifyEvent(int mask, int cookie, const QString &path) //qDebug() << cookie << " OTHER " << mask << " :" << path; } + foreach (QString pattern, _ignores) { + QRegExp regexp(pattern); + regexp.setPatternSyntax(QRegExp::Wildcard); + if (regexp.exactMatch(path)) { + qDebug() << "* Discarded" << path; + return; + } + } + _pendingPaths.append(path); slotProcessPaths(); } @@ -166,38 +216,45 @@ void FolderWatcher::slotProcessTimerTimeout() slotProcessPaths(); } +void FolderWatcher::slotPollTimerTimeout() +{ + qDebug() << "* Polling remote for changes"; + emit folderChanged(QStringList()); +} + void FolderWatcher::setProcessTimer() { if (!_processTimer->isActive()) { - qDebug() << "* Pending events will be processed in" << eventInterval() << "seconds. (" << _pendingPaths.size() << "events until now )"; - _processTimer->start(eventInterval() * 1000); + qDebug() << "* Pending events will be processed in" << eventInterval() << "seconds (" << QTime::currentTime().addSecs(eventInterval()).toString("HH:mm:ss") << ")." << _pendingPaths.size() << "events until now )"; } + _processTimer->start(eventInterval() * 1000); } void FolderWatcher::slotProcessPaths() { QTime eventTime = QTime::currentTime(); + QTime lastEventTime = _lastEventTime; + _lastEventTime = eventTime; - if (!eventsEnabled()) - return; - - if (!_lastEventTime.isNull() && (_lastEventTime.secsTo(eventTime) < eventInterval())) { - //qDebug() << "`-> Last event happened less than " << eventInterval() << " seconds ago..."; - // schedule a forced queue cleanup later + // if the events are disabled or the last event happened + // recently eg: copying lot of ifles + if (!eventsEnabled() || + ( !lastEventTime.isNull() && + (lastEventTime.secsTo(eventTime) < eventInterval()) )) + { + // in case this is the last file from a bulk copy + // set the process timer again so that we process the + // queue we are not processing now setProcessTimer(); return; } - // if the events will be processed because changed files and not - // because a forced update, stop any timer. - if (_processTimer->isActive()) - _processTimer->stop(); - - _lastEventTime = eventTime; QStringList notifyPaths(_pendingPaths); _pendingPaths.clear(); + //qDebug() << lastEventTime << eventTime; qDebug() << " * Notify " << notifyPaths.size() << " changed items"; + emit folderChanged(notifyPaths); } diff --git a/src/mirall/folderwatcher.h b/src/mirall/folderwatcher.h index 5c3442747..6ca9acf03 100644 --- a/src/mirall/folderwatcher.h +++ b/src/mirall/folderwatcher.h @@ -3,6 +3,7 @@ #ifndef MIRALL_FOLDERWATCHER_H #define MIRALL_FOLDERWATCHER_H +#include #include #include #include @@ -42,6 +43,14 @@ public: */ QString root() const; + /** + * Add an ignore pattern that will not be + * notified + * + * You can use wildcards + */ + void addIgnore(const QString &pattern); + /** * If true, folderChanged() events are sent * at least as often as eventInterval() seconds. @@ -67,6 +76,19 @@ public: */ void setEventInterval(int seconds); + /** + * The minimum amounts of seconds to wait before + * doing a full sync to see if the remote changed + */ + int pollInterval() const; + + /** + * Sets minimum amounts of seconds that will separate + * poll intervals + */ + void setPollInterval(int seconds); + + signals: /** * Emitted when one of the paths is changed @@ -81,20 +103,29 @@ protected slots: void slotAddFolderRecursive(const QString &path); // called when the manually process timer triggers void slotProcessTimerTimeout(); + void slotPollTimerTimeout(); void slotProcessPaths(); + private: bool _eventsEnabled; int _eventInterval; + int _pollInterval; + INotify *_inotify; QString _root; // paths pending to notified QStringList _pendingPaths; + QTimer *_processTimer; + // poll timer for remote syncs + QTimer *_pollTimer; + QTime _lastEventTime; // to cancel events that belong to the same action int _lastMask; QString _lastPath; + QStringList _ignores; }; } diff --git a/src/mirall/inotify.cpp b/src/mirall/inotify.cpp index 737492c46..f47e39857 100644 --- a/src/mirall/inotify.cpp +++ b/src/mirall/inotify.cpp @@ -11,6 +11,7 @@ http://www.gnu.org/licenses/gpl.txt . #include #include #include +#include #include #include "inotify.h" @@ -24,6 +25,7 @@ namespace Mirall { // Allocate space for static members of class. int INotify::s_fd; INotify::INotifyThread* INotify::s_thread; +QMutex INotify::INotifyThread::s_mutex; //INotify::INotify(int wd) : _wd(wd) //{ @@ -46,6 +48,7 @@ INotify::~INotify() void INotify::addPath(const QString &path) { + //QMutexLocker locker(&INotifyThread::s_mutex); // Add an inotify watch. path.toAscii().constData(); @@ -58,6 +61,7 @@ void INotify::addPath(const QString &path) void INotify::removePath(const QString &path) { + QMutexLocker locker(&INotifyThread::s_mutex); // Remove the inotify watch. inotify_rm_watch(s_fd, _wds[path]); _wds.remove(path); @@ -72,6 +76,7 @@ void INotify::INotifyThread::unregisterForNotification(INotify* notifier) { //_map.remove(notifier->_wd); + //QMutexLocker locker(&INotifyThread::s_mutex); QHash::iterator it; for (it = _map.begin(); it != _map.end(); ++it) { if (it.value() == notifier) @@ -82,14 +87,18 @@ INotify::INotifyThread::unregisterForNotification(INotify* notifier) void INotify::INotifyThread::registerForNotification(INotify* notifier, int wd) { + //QMutexLocker locker(&INotifyThread::s_mutex); _map[wd] = notifier; } void INotify::fireEvent(int mask, int cookie, int wd, char* name) { - QString path; - foreach (path, _wds.keys(wd)) + //qDebug() << "****" << name; + //QMutexLocker locker(&INotifyThread::s_mutex); + + QStringList paths(_wds.keys(wd)); + foreach (QString path, paths) emit notifyEvent(mask, cookie, path + "/" + QString::fromUtf8(name)); } @@ -170,12 +179,25 @@ INotify::INotifyThread::run() continue; } n = _map[event->wd]; + // dont allow addPath removePath clash here + { + //QThread::msleep(100); + // fire event + if (event->len > 0) { + //QMutexLocker locker(&s_mutex); + if (n) + n->fireEvent(event->mask, event->cookie, event->wd, event->name); + else + { + qWarning() << "n is NULL"; + } + + } + + // increment counter + i += sizeof(struct inotify_event) + event->len; + } - // fire event - if (event->len > 0) - n->fireEvent(event->mask, event->cookie, event->wd, event->name); - // increment counter - i += sizeof(struct inotify_event) + event->len; } } } diff --git a/src/mirall/inotify.h b/src/mirall/inotify.h index f1d519ecc..22697f116 100644 --- a/src/mirall/inotify.h +++ b/src/mirall/inotify.h @@ -11,7 +11,8 @@ http://www.gnu.org/licenses/gpl.txt . #define MIRALL_INOTIFY_H #include -#include +#include +#include #include #include @@ -48,6 +49,9 @@ private: ~INotifyThread(); void registerForNotification(INotify*, int); void unregisterForNotification(INotify*); + // fireEvent happens from the inotify thread + // but addPath comes from the main thread + static QMutex s_mutex; protected: void run(); private: @@ -62,7 +66,7 @@ private: // the mask is shared for all paths int _mask; - QHash _wds; + QMap _wds; }; } diff --git a/src/mirall/syncresult.h b/src/mirall/syncresult.h new file mode 100644 index 000000000..dc8c2dfcd --- /dev/null +++ b/src/mirall/syncresult.h @@ -0,0 +1,21 @@ +#ifndef MIRALL_SYNCRESULT_H +#define MIRALL_SYNCRESULT_H + +#include + +namespace Mirall +{ + +class SyncResult +{ +public: + SyncResult(); + ~SyncResult(); +private: + QStringList _deletedSource; + QStringList _deletedDestination; +}; + +} + +#endif diff --git a/src/mirall/unisonfolder.cpp b/src/mirall/unisonfolder.cpp index 1919a7633..828a1ce8a 100644 --- a/src/mirall/unisonfolder.cpp +++ b/src/mirall/unisonfolder.cpp @@ -1,7 +1,9 @@ #include +#include #include #include -#include +#include + #include "mirall/unisonfolder.h" namespace Mirall { @@ -56,7 +58,7 @@ void UnisonFolder::startSync(const QStringList &pathList) args << "-ui" << "text"; args << "-auto" << "-batch"; - //args << "-confirmbigdel"; + args << "-confirmbigdel=false"; // only use -path in after a full synchronization // already happened, which we do only on the first @@ -72,6 +74,7 @@ void UnisonFolder::startSync(const QStringList &pathList) args << path(); args << secondPath(); + qDebug() << " * Unison: will use" << pathList.size() << "path arguments"; _unison->start(program, args); } @@ -86,17 +89,35 @@ void UnisonFolder::slotStarted() void UnisonFolder::slotFinished(int exitCode, QProcess::ExitStatus exitStatus) { qDebug() << " * Unison process finished with status" << exitCode; + + //if (exitCode != 0) { + qDebug() << _lastOutput; + //} + + // parse a summary from here: + //[BGN] Copying zw.png from //piscola//space/store/folder1 to /space/mirall/folder1 + //[BGN] Deleting gn.png from /space/mirall/folder1 + //[END] Deleting gn.png + + // from stderr: + //Reconciling changes + // <---- new file Package.h + + _lastOutput.clear(); + emit syncFinished(); } void UnisonFolder::slotReadyReadStandardOutput() { - qDebug() << _unison->readAllStandardOutput();; + QTextStream stream(&_lastOutput); + stream << _unison->readAllStandardOutput();; } void UnisonFolder::slotReadyReadStandardError() { - //qDebug() << _unison->readAllStandardError();; + QTextStream stream(&_lastOutput); + stream << _unison->readAllStandardError();; } void UnisonFolder::slotStateChanged(QProcess::ProcessState state) diff --git a/src/mirall/unisonfolder.h b/src/mirall/unisonfolder.h index 82bc46e4a..8d6a8ba10 100644 --- a/src/mirall/unisonfolder.h +++ b/src/mirall/unisonfolder.h @@ -36,6 +36,9 @@ private: QProcess *_unison; QString _secondPath; int _syncCount; + + QString _lastOutput; + }; } -- cgit v1.2.3