Merge branch 'feature/disable-file-scanner'

5 files changed, 51 insertions, 7 deletions

diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
index 45450e2a5..5f3a7c799 100644
--- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
@@ -604,6 +604,18 @@ namespace Duplicati.Library.Main.Database
             return lastFilesetId;

         }

 

+        internal Tuple<long, long> GetLastBackupFileCountAndSize()

+        {

+            using (var cmd = m_connection.CreateCommand())

+            {

+                var lastFilesetId = cmd.ExecuteScalarInt64(@"SELECT ""ID"" FROM ""Fileset"" ORDER BY ""Timestamp"" DESC LIMIT 1");

+                var count = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""File"" INNER JOIN ""FilesetEntry"" ON ""File"".""ID"" = ""FilesetEntry"".""FileID"" WHERE ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);

+                var size = cmd.ExecuteScalarInt64(@"SELECT SUM(""Blockset"".""Length"") FROM ""File"", ""FilesetEntry"", ""Blockset"" WHERE ""File"".""ID"" = ""FilesetEntry"".""FileID"" AND ""File"".""BlocksetID"" = ""Blockset"".""ID"" AND ""FilesetEntry"".""FilesetID"" = ? AND ""File"".""BlocksetID"" NOT IN (?, ?)", -1, lastFilesetId, FOLDER_BLOCKSET_ID, SYMLINK_BLOCKSET_ID);

+

+                return new Tuple<long, long>(count, size);

+            }

+        }

+

         internal void UpdateChangeStatistics(BackupResults results)

         {

             using(var cmd = m_connection.CreateCommand())

diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs
index 78c02a64e..0aa895ae0 100644
--- a/Duplicati/Library/Main/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/Operation/BackupHandler.cs
@@ -732,14 +732,23 @@ namespace Duplicati.Library.Main.Operation
                     {

                         using(var snapshot = GetSnapshot(sources, m_options, m_result))

                         {

-                            // Start parallel scan

+                            // Start parallel scan, or use the database

                             if (m_options.ChangedFilelist == null || m_options.ChangedFilelist.Length < 1)

                             {

-                                parallelScanner = new System.Threading.Thread(CountFilesThread) {

-                                    Name = "Read ahead file counter",

-                                    IsBackground = true

-                                };

-                                parallelScanner.Start(snapshot);

+                                if (m_options.DisableFileScanner)

+                                {

+                                    var d = m_database.GetLastBackupFileCountAndSize();

+                                    m_result.OperationProgressUpdater.UpdatefileCount(d.Item1, d.Item2, true);

+                                }

+                                else

+                                {

+                                    parallelScanner = new System.Threading.Thread(CountFilesThread)

+                                    {

+                                        Name = "Read ahead file counter",

+                                        IsBackground = true

+                                    };

+                                    parallelScanner.Start(snapshot);

+                                }

                             }

 

                             string lasttempfilelist = null;

diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index c917ca3e2..465dfcbf7 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -256,7 +256,8 @@ namespace Duplicati.Library.Main
                     "exclude-files-attributes",

                     "compression-extension-file",

                     "full-remote-verification",

-                    "disable-synthetic-filelist"

+                    "disable-synthetic-filelist",

+                    "disable-file-scanner"

                 };

             }

         }

@@ -530,6 +531,7 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("disable-piped-streaming", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablepipingShort, Strings.Options.DisablepipingLong, "false"),

 

                     new CommandLineArgument("auto-vacuum", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AutoVacuumShort, Strings.Options.AutoVacuumLong, "false"),

+                    new CommandLineArgument("disable-file-scanner", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilescannerShort, Strings.Options.DisablefilescannerLong, "false"),

                 });

 

                 return lst;

@@ -1792,6 +1794,15 @@ namespace Duplicati.Library.Main
         }

 

         /// <summary>

+        /// Gets a flag indicating if the local filescanner should be disabled

+        /// </summary>

+        /// <value><c>true</c> if the filescanner should be disabled; otherwise, <c>false</c>.</value>

+        public bool DisableFileScanner

+        {

+            get { return Library.Utility.Utility.ParseBoolOption(m_options, "disable-file-scanner"); }

+        }

+

+        /// <summary>

         /// Gets the threshold for when log data should be cleaned

         /// </summary>

         public DateTime LogRetention

diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index eecba8c0b..274b7b6c4 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -237,6 +237,8 @@ namespace Duplicati.Library.Main.Strings
         public static string AllowfullremovalLong { get { return LC.L(@"By default, the last fileset cannot be removed. This is a safeguard to make sure that all remote data is not deleted by a configuration mistake. Use this flag to disable that protection, such that all filesets can be deleted."); } }

         public static string AutoVacuumShort { get { return LC.L(@"Allow automatic rebuilding of local database to save space."); } }

         public static string AutoVacuumLong { get { return LC.L(@"Some operations that manipulate the local database leave unused entries behind. These entries are not deleted from a hard drive until a VACUUM operation is run. This operation saves disk space in the long run but needs to temporarily create a copy of all valid entries in the database. Setting this to true will allow Duplicati to perform VACUUM operations at its discretion."); } }

+        public static string DisablefilescannerShort { get { return LC.L(@"Disable the read-ahead scanner"); } }

+        public static string DisablefilescannerLong { get { return LC.L(@"When this flag is enabled, the scanner that computes the size of source files is disabled, and instead the reported size is read from the database. Using this flag can speed up the backup by reducing disk access, but will give a less accurate progress indicator."); } }

     }

 

     internal static class Common

diff --git a/Duplicati/UnitTest/BorderTests.cs b/Duplicati/UnitTest/BorderTests.cs
index 96113dce6..27bbf2f0b 100644
--- a/Duplicati/UnitTest/BorderTests.cs
+++ b/Duplicati/UnitTest/BorderTests.cs
@@ -34,6 +34,16 @@ namespace Duplicati.UnitTest
 

         [Test]

         [Category("Border")]

+        public void Run10kNoProgress()

+        {

+            PrepareSourceData();

+            RunCommands(1024 * 10, modifyOptions: opts => { 

+                opts["disable-file-scanner"] = "true"; 

+            });

+        }

+

+        [Test]

+        [Category("Border")]

         public void Run10k()

         {

             PrepareSourceData();