Re-introduced the in-memory block-cache. This time keeping it simple with a check for size-induced collisions.

3 files changed, 57 insertions, 8 deletions

diff --git a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
index 568e5e3bd..d4d7bea62 100644
--- a/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalBackupDatabase.cs
@@ -85,6 +85,7 @@ namespace Duplicati.Library.Main.Database
         private readonly System.Data.IDbCommand m_insertfileOperationCommand;

         

         private PathLookupHelper<PathEntryKeeper> m_pathLookup;

+        private Dictionary<string, long> m_blockCache;

         

         private long m_filesetId;

 

@@ -245,9 +246,30 @@ namespace Duplicati.Library.Main.Database
                 var tc = cmd.ExecuteScalarInt64(@"SELECT COUNT(*) FROM ""Remotevolume"" WHERE ""ID"" IN (SELECT DISTINCT ""VolumeID"" FROM ""Block"") AND ""State"" NOT IN (?, ?, ?, ?);", 0, RemoteVolumeState.Temporary.ToString(), RemoteVolumeState.Uploading.ToString(), RemoteVolumeState.Uploaded.ToString(), RemoteVolumeState.Verified.ToString());

                 if (tc > 0)

                     throw new InvalidDataException("Detected blocks that are not reachable in the block table");

-

-            }

-        }

+            }
+
+			if (options.UseBlockCache)
+			{

+                string failedhash = null;

+                try

+                {
+                    var cache = new Dictionary<string, long>();
+                    using (var cmd = m_connection.CreateCommand())
+                    {
+                        cmd.CommandText = @"SELECT ""Hash"", ""Size"" From ""Block""";
+                        using (var rd = cmd.ExecuteReader())
+                            while (rd.Read())
+                                cache.Add(failedhash = rd.ConvertValueToString(0), rd.ConvertValueToInt64(1));
+                    }

+                    m_blockCache = cache;

+                }

+                catch (Exception ex)

+                {

+                    Logging.Log.WriteMessage(string.Format("Failed to create block cache, this could mean you have hash collisions in your table, the hash that failed is {0}. Error message: {1}.", failedhash, ex.Message), Logging.LogMessageType.Warning);
+					Logging.Log.WriteMessage(string.Format("Disabling block cache due to error"), Logging.LogMessageType.Warning);
+				}
+			}
+		}

 

         /// <summary>

         /// Adds a block to the local database, returning a value indicating if the value presents a new block

@@ -257,6 +279,17 @@ namespace Duplicati.Library.Main.Database
         /// <returns>True if the block should be added to the current output</returns>

         public bool AddBlock (string key, long size, long volumeid, System.Data.IDbTransaction transaction = null)

         {

+            long exsize;

+

+            if (m_blockCache != null && m_blockCache.TryGetValue(key, out exsize))

+            {

+                if (exsize == size)

+                    return false;

+

+                Logging.Log.WriteMessage(string.Format("Found hash collision on {0}, sizes {1} vs {2}. Disabling cache from now on.", key, size, exsize), Logging.LogMessageType.Warning);

+                m_blockCache = null;

+            }

+

             m_findblockCommand.Transaction = transaction;

             m_findblockCommand.SetParameterValue(0, key);

             m_findblockCommand.SetParameterValue(1, size);

@@ -269,6 +302,8 @@ namespace Duplicati.Library.Main.Database
                 m_insertblockCommand.SetParameterValue(1, volumeid);

                 m_insertblockCommand.SetParameterValue(2, size);

                 r = m_insertblockCommand.ExecuteScalarInt64();

+                if (m_blockCache != null)

+                    m_blockCache.Add(key, size);

                 return true;

             }

             else

diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index 987024695..e82403930 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -480,8 +480,9 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("skip-metadata", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkipmetadataShort, Strings.Options.SkipmetadataLong, "false"),

                     new CommandLineArgument("restore-permissions", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RestorepermissionsShort, Strings.Options.RestorepermissionsLong, "false"),

                     new CommandLineArgument("skip-restore-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SkiprestoreverificationShort, Strings.Options.SkiprestoreverificationLong, "false"),

-                    new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"),

-                    new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong),

+                    new CommandLineArgument("disable-filepath-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablefilepathcacheShort, Strings.Options.DisablefilepathcacheLong, "true"),
+					new CommandLineArgument("use-block-cache", CommandLineArgument.ArgumentType.Boolean, Strings.Options.UseblockcacheShort, Strings.Options.UseblockcacheLong, "false"),
+					new CommandLineArgument("changed-files", CommandLineArgument.ArgumentType.Path, Strings.Options.ChangedfilesShort, Strings.Options.ChangedfilesLong),

                     new CommandLineArgument("deleted-files", CommandLineArgument.ArgumentType.Path, Strings.Options.DeletedfilesShort, Strings.Options.DeletedfilesLong("changed-files")),

                     new CommandLineArgument("disable-synthetic-filelist", CommandLineArgument.ArgumentType.Boolean, Strings.Options.DisablesyntheticfilelistShort, Strings.Options.DisablesyntehticfilelistLong, "false"),

 

@@ -1442,7 +1443,7 @@ namespace Duplicati.Library.Main
         }

 

         /// <summary>

-        /// Gets the file hash size

+        /// Flag indicating if the filepath cache is disabled

         /// </summary>

         public bool UseFilepathCache

         {

@@ -1453,6 +1454,17 @@ namespace Duplicati.Library.Main
                 return !Library.Utility.Utility.ParseBool(s, true);

             }

         }

+

+        /// <summary>

+        /// Flag indicating if the in-memory block cache is used

+        /// </summary>

+        public bool UseBlockCache

+        {

+            get

+            {

+                return Library.Utility.Utility.ParseBoolOption(m_options, "use-block-cache");
+			}

+        }

         

         

         /// <summary>

diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index eb8947bb3..775746d37 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -142,8 +142,10 @@ namespace Duplicati.Library.Main.Strings
         public static string FilehashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } }

         public static string FilehashlookupsizeShort { get { return LC.L(@"Memory used by the file hash"); } }

         public static string DisablefilepathcacheLong { get { return LC.L(@"This option can be used to reduce the memory footprint by not keeping paths and modification timestamps in memory"); } }

-        public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } }

-        public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }

+        public static string DisablefilepathcacheShort { get { return LC.L(@"Reduce memory footprint by disabling in-memory lookups"); } }
+		public static string UseblockcacheShort { get { return LC.L(@"This option can be used to increase speed in exchange for extra memory use."); } }
+		public static string UseblockcacheLong { get { return LC.L(@"Store an in-memory block cache"); } }
+		public static string StoremetadataLong { get { return LC.L(@"Stores metadata, such as file timestamps and attributes. This increases the required storage space as well as the processing time."); } }

         public static string StoremetadataShort { get { return LC.L(@"Enables storing file metadata"); } }

         public static string StoremetadataDeprecated { get { return LC.L(@"This option is no longer used as metadata is now stored by default"); } }

         public static string MetadatahashlookupsizeLong { get { return LC.L(@"A fragment of memory is used to reduce database lookups. You should not change this value unless you get warnings in the log."); } }