Merge branch 'master' into concurrent_processing

9 files changed, 113 insertions, 57 deletions

diff --git a/Duplicati/CommandLine/help.txt b/Duplicati/CommandLine/help.txt
index 5d9f01316..d4976e082 100644
--- a/Duplicati/CommandLine/help.txt
+++ b/Duplicati/CommandLine/help.txt
@@ -12,8 +12,8 @@
 See duplicati.commandline.exe help <topic> for more information.

   General: example, changelog

   Commands: backup, find, restore, delete, compact, test, compare, purge, vacuum

-  Reparing: repair, affected, list-broken-files, purge-broken-files

-  Debugging: debug, logging, create-report, test-filters, system-info, send-mail

+  Repair: repair, affected, list-broken-files, purge-broken-files

+  Debug: debug, logging, create-report, test-filters, system-info, send-mail

   Targets: %BACKENDS%

   Modules: %ENCRYPTIONMODULES%, %COMPRESSIONMODULES%, %GENERICMODULES%

   Formats: date, time, size, encryption, compression

diff --git a/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs b/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs
index a88cf21d6..cf6fe2a32 100644
--- a/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs
+++ b/Duplicati/Library/Modules/Builtin/ResultSerialization/DuplicatiFormatSerializer.cs
@@ -3,6 +3,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Text;
+using Duplicati.Library.Interface;
 
 namespace Duplicati.Library.Modules.Builtin.ResultSerialization
 {
@@ -86,7 +87,13 @@ namespace Duplicati.Library.Modules.Builtin.ResultSerialization
             }
             else
             {
-                Utility.Utility.PrintSerializeObject(result, sb);
+                var ignore = new string[] { 
+                    nameof(IBasicResults.Warnings),
+                    nameof(IBasicResults.Errors),
+                    nameof(IBasicResults.Messages)
+                };
+
+                Utility.Utility.PrintSerializeObject(result, sb, (p, o) => !ignore.Contains(p.Name));
             }
 
             if (additional != null && additional.Count > 0)
diff --git a/Duplicati/Library/Utility/FilterGroups.cs b/Duplicati/Library/Utility/FilterGroups.cs
index c4a4cee45..e9e05a887 100644
--- a/Duplicati/Library/Utility/FilterGroups.cs
+++ b/Duplicati/Library/Utility/FilterGroups.cs
@@ -75,7 +75,7 @@ namespace Duplicati.Library.Utility
         /// <summary>

         /// In addition to the default names from the enums, these alternate / shorter names are also available.

         /// </summary>

-        private static Dictionary<string, FilterGroup> filterGroupAliases = new Dictionary<string, FilterGroup>(StringComparer.OrdinalIgnoreCase)

+        private static readonly Dictionary<string, FilterGroup> filterGroupAliases = new Dictionary<string, FilterGroup>(StringComparer.OrdinalIgnoreCase)

             {

                 { "System", FilterGroup.SystemFiles },

                 { "Sys", FilterGroup.SystemFiles },

@@ -108,6 +108,16 @@ namespace Duplicati.Library.Utility
             };

 

         /// <summary>

+        /// Regex escaped string for the AltDirectorySeparatorChar

+        /// </summary>

+        private static readonly string RegexEscapedAltDirectorySeparatorChar = System.Text.RegularExpressions.Regex.Escape(System.IO.Path.AltDirectorySeparatorChar.ToString());

+

+        /// <summary>

+        /// Regex escaped string for the DirectorySeparatorChar

+        /// </summary>

+        private static readonly string RegexEscapedDirectorySeparatorChar = System.Text.RegularExpressions.Regex.Escape(System.IO.Path.DirectorySeparatorChar.ToString());

+

+        /// <summary>

         /// Gets the list of alternate aliases which can refer to this group.

         /// </summary>

         /// <param name="group">Filter group</param>

@@ -277,7 +287,7 @@ namespace Duplicati.Library.Utility
                 // TODO: The control_dir_v2 might be under a different path for OEM branded instances.

                 // However, the AppName is loaded and controlled by the AutoUpdater assembly, which we can't reference here without an ugly circular dependency or dependency injection.

                 // What is the best way to solve this?

-                yield return FilterGroups.CreateWildcardFilter(@"*/Duplicati/control_dir_v2"); // Duplicati uses this directory to store lock files and communicate with other processes.

+                yield return FilterGroups.CreateWildcardFilter(@"*/Duplicati/control_dir_v2/"); // Duplicati uses this directory to store lock files and communicate with other processes.

                 yield return FilterGroups.CreateWildcardFilter(@"*/Google/Chrome/*cache*");

                 yield return FilterGroups.CreateWildcardFilter(@"*/Google/Chrome/*LOCK*"); // Chrome appears to lock various files under it's settings folder using files named 'LOCK' or 'lockfile'

                 yield return FilterGroups.CreateWildcardFilter(@"*/Google/Chrome/*Current*"); // 'Current Session' and 'Current Tabs' appear to be locked while running Chrome

@@ -335,18 +345,18 @@ namespace Duplicati.Library.Utility
                 yield return FilterGroups.CreateWildcardFilter(@"?:/Config.Msi*"); // https://github.com/duplicati/duplicati/issues/2886

                 yield return FilterGroups.CreateWildcardFilter(@"*/Recent/");

                 yield return FilterGroups.CreateWildcardFilter(@"?:/autoexec.bat");

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.System);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.SystemX86);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.Windows);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.Recent);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.System);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.SystemX86);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.Windows);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.Recent);

 

-                var windir = Environment.GetFolderPath(Environment.SpecialFolder.Windows);

+                var windir = FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.Windows);

                 if (!string.IsNullOrWhiteSpace(windir))

                 {

                     yield return windir;

 

                     // Also exclude "C:\Windows.old\"

-                    yield return windir.TrimEnd('\\', '/') + ".old";

+                    yield return Utility.AppendDirSeparator(windir.TrimEnd('\\', '/') + ".old");

                 }

             }

 

@@ -375,9 +385,9 @@ namespace Duplicati.Library.Utility
                 yield return FilterGroups.CreateWildcardFilter(@"*/Temporary Internet Files/");

                 yield return FilterGroups.CreateWildcardFilter(@"*/Thumbs.db");

 

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.History);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.InternetCache);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.Cookies);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.History);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.InternetCache);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.Cookies);

 

             }

 

@@ -394,12 +404,12 @@ namespace Duplicati.Library.Utility
 

             if (group.HasFlag(FilterGroup.Applications))

             {

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.AdminTools);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.CommonProgramFiles);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.CommonProgramFilesX86);

-                yield return Environment.GetFolderPath(Environment.SpecialFolder.CommonAdminTools);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.ProgramFiles);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.ProgramFilesX86);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.AdminTools);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.CommonProgramFiles);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.CommonProgramFilesX86);

+                yield return FilterGroups.CreateSpecialFolderFilter(Environment.SpecialFolder.CommonAdminTools);

             }

         }

 

@@ -555,6 +565,31 @@ namespace Duplicati.Library.Utility
         }

 

         /// <summary>

+        /// Creates a filter for a special folder

+        /// </summary>

+        /// <param name="specialFolder">Special folder</param>

+        /// <returns>Special folder filter</returns>

+        private static string CreateSpecialFolderFilter(Environment.SpecialFolder specialFolder)

+        {

+            string folderPath = Environment.GetFolderPath(specialFolder);

+            if (!string.IsNullOrEmpty(folderPath))

+            {

+                // Note that this also replaces alternate directory separators with regular ones

+                string filter = FilterGroups.CreateWildcardFilter(folderPath);

+

+                // Duplicati matches filters against folder paths exactly.

+                // Meaning a filter for 'C:\Windows' won't match 'C:\Windows\'.

+                // So this makes sure special folder's filter's have a trailing directory separator.

+                // (Alternatively, this could append '*' to all folder filters.)

+                return Utility.AppendDirSeparator(filter);

+            }

+            else

+            {

+                return null;

+            }

+        }

+

+        /// <summary>

         /// Creates a wildcard filter

         /// </summary>

         /// <param name="filter">Filter text</param>

@@ -575,9 +610,7 @@ namespace Duplicati.Library.Utility
         {

             // Create a filter with the given name.

             // However, in order to match paths correctly, the directory separators need to be normalized to match the system default.

-            string escapedAlt = System.Text.RegularExpressions.Regex.Escape(System.IO.Path.AltDirectorySeparatorChar.ToString());

-            string escaped = System.Text.RegularExpressions.Regex.Escape(System.IO.Path.DirectorySeparatorChar.ToString());

-            return "[" + filter.Replace(escapedAlt, escaped) + "]";

+            return "[" + filter.Replace(FilterGroups.RegexEscapedAltDirectorySeparatorChar, FilterGroups.RegexEscapedDirectorySeparatorChar) + "]";

         }

 

         /// <summary>

diff --git a/Duplicati/Server/Database/ServerSettings.cs b/Duplicati/Server/Database/ServerSettings.cs
index 2ba5d3716..bafb09355 100644
--- a/Duplicati/Server/Database/ServerSettings.cs
+++ b/Duplicati/Server/Database/ServerSettings.cs
@@ -188,10 +188,6 @@ namespace Duplicati.Server.Database
         {

             get

             {

-                var tp = m_values[CONST.IS_FIRST_RUN];

-                if (string.IsNullOrEmpty(tp))

-                    return true;

-

                 return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.IS_FIRST_RUN);

             }

             set

@@ -202,29 +198,25 @@ namespace Duplicati.Server.Database
             }

         }
 
-		public bool HasAskedForPasswordProtection
-		{
-			get
-			{
-                var tp = m_values[CONST.HAS_ASKED_FOR_PASSWORD_PROTECTION];
-				if (string.IsNullOrEmpty(tp))
-					return true;
-
-				return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.HAS_ASKED_FOR_PASSWORD_PROTECTION);
-			}
-			set
-			{
-				lock (m_connection.m_lock)
-					m_values[CONST.HAS_ASKED_FOR_PASSWORD_PROTECTION] = value.ToString();
-				SaveSettings();
-			}
-		}
+        public bool HasAskedForPasswordProtection
+        {
+            get
+            {

+                return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.HAS_ASKED_FOR_PASSWORD_PROTECTION);

+            }
+            set
+            {
+                lock (m_connection.m_lock)
+                    m_values[CONST.HAS_ASKED_FOR_PASSWORD_PROTECTION] = value.ToString();
+                SaveSettings();
+            }
+        }
 
-		public bool UnackedError

+        public bool UnackedError

         {

             get

             {

-                return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.UNACKED_ERROR);

+                return Duplicati.Library.Utility.Utility.ParseBool(m_values[CONST.UNACKED_ERROR], false);

             }

             set

             {

@@ -238,7 +230,7 @@ namespace Duplicati.Server.Database
         {

             get

             {

-                return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.UNACKED_WARNING);

+                return Duplicati.Library.Utility.Utility.ParseBool(m_values[CONST.UNACKED_WARNING], false);

             }

             set

             {

@@ -247,11 +239,12 @@ namespace Duplicati.Server.Database
                 SaveSettings();

             }

         }

+

         public bool ServerPortChanged

         {

             get

             {

-                return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.SERVER_PORT_CHANGED);

+                return Duplicati.Library.Utility.Utility.ParseBool(m_values[CONST.SERVER_PORT_CHANGED], false);

             }

             set

             {

@@ -535,10 +528,7 @@ namespace Duplicati.Server.Database
         {

             get

             {

-                if (m_values.ContainsKey(CONST.HAS_FIXED_INVALID_BACKUPID) && string.IsNullOrWhiteSpace(m_values[CONST.HAS_FIXED_INVALID_BACKUPID]))

-                    return false;

-                else

-                    return Duplicati.Library.Utility.Utility.ParseBoolOption(m_values, CONST.HAS_FIXED_INVALID_BACKUPID);

+                return Duplicati.Library.Utility.Utility.ParseBool(m_values[CONST.HAS_FIXED_INVALID_BACKUPID], false);

             }

             set

             {

diff --git a/Duplicati/Server/WebServer/RESTMethods/Backups.cs b/Duplicati/Server/WebServer/RESTMethods/Backups.cs
index 87ca5bbc2..15a9c3fef 100644
--- a/Duplicati/Server/WebServer/RESTMethods/Backups.cs
+++ b/Duplicati/Server/WebServer/RESTMethods/Backups.cs
@@ -91,9 +91,15 @@ namespace Duplicati.Server.WebServer.RESTMethods
                                 ipx = Serializer.Deserialize<Serializable.ImportExportStructure>(sr);

                         }

                     }

-                    if (!import_metadata) {

+

+                    if (ipx.Backup == null)

+                        throw new Exception("No backup found in document");

+

+                    if (ipx.Backup.Metadata == null)

+                        ipx.Backup.Metadata = new Dictionary<string, string>();

+

+                    if (!import_metadata) 

                         ipx.Backup.Metadata.Clear();

-                    }

 

                     ipx.Backup.ID = null;

                     ((Database.Backup)ipx.Backup).DBPath = null;

diff --git a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js
index fca2902db..0aa3b2f3b 100644
--- a/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js
+++ b/Duplicati/Server/webroot/ngax/scripts/controllers/EditBackupController.js
@@ -41,7 +41,7 @@ backupApp.controller('EditBackupController', function ($rootScope, $scope, $rout
         else if ((passphrase || '') == '')
             scope.PassphraseScore = '';
         else
-            scope.PassphraseScore = (zxcvbn(passphrase) || {'score': -1}).score;
+            scope.PassphraseScore = (zxcvbn(passphrase.substring(0, 100)) || {'score': -1}).score;
 
         scope.PassphraseScoreString = strengthMap[scope.PassphraseScore];
     }
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/ServerStatus.js b/Duplicati/Server/webroot/ngax/scripts/services/ServerStatus.js
index e81e00a07..948e09b6e 100644
--- a/Duplicati/Server/webroot/ngax/scripts/services/ServerStatus.js
+++ b/Duplicati/Server/webroot/ngax/scripts/services/ServerStatus.js
@@ -54,7 +54,7 @@ backupApp.service('ServerStatus', function($rootScope, $timeout, AppService, App
             'Restore_PostRestoreVerify': gettextCatalog.getString('Verifying restored files ...'),
             'Restore_Complete': gettextCatalog.getString('Restore Complete!'),
             'Recreate_Running': gettextCatalog.getString('Recreating database ...'),
-            'Repair_Running': gettextCatalog.getString('Reparing database ...'),
+            'Repair_Running': gettextCatalog.getString('Repairing database ...'),
             'Verify_Running': gettextCatalog.getString('Verifying files...'),
             'BugReport_Running': gettextCatalog.getString('Creating bug report ...'),
             'Delete_Listing': gettextCatalog.getString('Listing remote files ...'),
diff --git a/Updates/build_version.txt b/Updates/build_version.txt
index b8626c4cf..7ed6ff82d 100644
--- a/Updates/build_version.txt
+++ b/Updates/build_version.txt
@@ -1 +1 @@
-4
+5
diff --git a/changelog.txt b/changelog.txt
index 57c36bff5..80b16d30f 100644
--- a/changelog.txt
+++ b/changelog.txt
@@ -1,3 +1,23 @@
+2018-04-13 - 2.0.3.5_canary_2018-04-13
+==========
+* Improved progress bar messsages and layout, thanks @SanduRajapakse
+* Improved icon status images, thanks @dbddhkpde
+* Code and test cleanups, thanks @warwickmm
+* Fixed an issue with rclone backend ignoring options, thanks @Bruceforce
+* Added a fix for browsers with scripting disabled, thanks @Pectojin
+* Added a button to dismiss all messages, thanks @Pectojin
+* Added KeyStone v3 support to OpenStack backend, thanks @epol
+* Updated translations, thanks to all translators!
+* Fixed an issue that caused large log data to accumulate and break sending report/email status
+* Added support for fine-grained control over how log data is reported via email, http and Jabber/XMPP
+* Added support for sending JSON formatted data with the http report module
+* Fixed an issue with MS Graph authentication, thanks @tygill
+* Fixed a performance issue during file scanning, thanks @ltfish
+* Added support for serializing results into json for all report modules and the run-script module as well, thanks @StephenGregory
+* Added filter groups and a UI for it, thanks @tygill
+* Fixed an issue where some paths were not reported via test-filters
+* Fixed some issues with handling internal server settings, thanks @warwickmm
+
 2018-04-02 - 2.0.3.4_canary_2018-04-02
 ==========
 * Added support for setting low IO priority during backups