Added automatic log cleanup

This fixes #1298

7 files changed, 79 insertions, 2 deletions

diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs
index 2149d9270..b8cfbbde8 100644
--- a/Duplicati/Library/Main/Database/LocalDatabase.cs
+++ b/Duplicati/Library/Main/Database/LocalDatabase.cs
@@ -934,6 +934,21 @@ namespace Duplicati.Library.Main.Database
 

             }

         }

+

+        public void PurgeLogData(DateTime threshold)

+        {

+            using(var tr = m_connection.BeginTransaction())

+            using(var cmd = m_connection.CreateCommand(tr))

+            {

+                var t = NormalizeDateTimeToEpochSeconds(threshold);

+                cmd.ExecuteNonQuery(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < ?", t);

+                cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < ?", t);

+

+                tr.Commit();

+            }

+            using(var cmd = m_connection.CreateCommand())

+                cmd.ExecuteNonQuery("VACUUM");

+        }

         

         public virtual void Dispose()

         {

diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs
index e9d622427..1bd24293c 100644
--- a/Duplicati/Library/Main/Operation/BackupHandler.cs
+++ b/Duplicati/Library/Main/Operation/BackupHandler.cs
@@ -738,6 +738,7 @@ namespace Duplicati.Library.Main.Operation
                         

                         m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete);

                         m_database.WriteResults();                    

+                        m_database.PurgeLogData(m_options.LogRetention);

                         return;

                     }

                 }

diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index 10703a50a..f8a9a3d97 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -69,6 +69,11 @@ namespace Duplicati.Library.Main
         private const int DEFAULT_KEEP_VERSIONS = 0;

 

         /// <summary>

+        /// The default threshold for purging log data

+        /// </summary>

+        private const string DEFAULT_LOG_RETENTION = "30D";

+

+        /// <summary>

         /// An enumeration that describes the supported strategies for an optimization

         /// </summary>

         public enum OptimizationStrategy

@@ -499,6 +504,9 @@ namespace Duplicati.Library.Main
                     new CommandLineArgument("allow-passphrase-change", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AllowpassphrasechangeShort, Strings.Options.AllowpassphrasechangeLong, "false"),

                     new CommandLineArgument("no-local-blocks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NolocalblocksShort, Strings.Options.NolocalblocksLong, "false"),

                     new CommandLineArgument("full-block-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.FullblockverificationShort, Strings.Options.FullblockverificationLong, "false"),

+

+                    new CommandLineArgument("log-retention", CommandLineArgument.ArgumentType.Timespan, Strings.Options.LogretentionShort, Strings.Options.LogretentionLong, DEFAULT_LOG_RETENTION),

+

                 });

 

                 return lst;

@@ -1719,6 +1727,21 @@ namespace Duplicati.Library.Main
             get { return Library.Utility.Utility.ParseBoolOption(m_options, "full-block-verification"); }

         }

 

+

+        /// <summary>

+        /// Gets the threshold for when log data should be cleaned

+        /// </summary>

+        public DateTime LogRetention

+        {

+            get

+            {

+                string pts;

+                if (!m_options.TryGetValue("log-retention", out pts))

+                    pts = DEFAULT_LOG_RETENTION;

+

+                return Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true);

+            }

+        }

         /// <summary>

         /// Gets a lookup table with compression hints, the key is the file extension with the leading period

         /// </summary>

diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index f3ef7db21..68c3f4410 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -199,6 +199,8 @@ namespace Duplicati.Library.Main.Strings
         public static string NolocalblocksLong { get { return LC.L(@"Duplicati will attempt to use data from source files to minimize the amount of downloaded data. Use this option to skip this optimization and only use remote data."); } }
         public static string FullblockverificationShort { get { return LC.L(@"Check block hashes"); } }
         public static string FullblockverificationLong { get { return LC.L(@"Use this option to increase verification by checking the hash of blocks read from a volume before patching restored files with the data."); } }
+        public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } }
+        public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } }
     }
 
     internal static class Foresthash 
diff --git a/Duplicati/Server/Database/Connection.cs b/Duplicati/Server/Database/Connection.cs
index 88ec66cb3..34a28fd5a 100644
--- a/Duplicati/Server/Database/Connection.cs
+++ b/Duplicati/Server/Database/Connection.cs
@@ -786,6 +786,30 @@ namespace Duplicati.Server.Database
             ticks -= ticks % TimeSpan.TicksPerSecond;

             return new DateTime(ticks, DateTimeKind.Utc);

         }

+

+        public void PurgeLogData(DateTime purgeDate)

+        {

+            var t = NormalizeDateTimeToEpochSeconds(purgeDate);

+

+            using(var tr = m_connection.BeginTransaction())

+            using(var cmd = m_connection.CreateCommand())

+            {

+                cmd.Transaction = tr;

+                cmd.CommandText = @"DELETE FROM ""ErrorLog"" WHERE ""Timestamp"" < ?";

+                cmd.Parameters.Add(cmd.CreateParameter());

+                ((System.Data.IDataParameter)cmd.Parameters[0]).Value = t;

+                cmd.ExecuteNonQuery();

+

+                tr.Commit();

+            }

+

+            using(var cmd = m_connection.CreateCommand())

+            {

+                cmd.CommandText = "VACUUM";

+                cmd.ExecuteNonQuery();

+            }

+

+        }

         

         private static long NormalizeDateTimeToEpochSeconds(DateTime input)

         {

@@ -869,7 +893,6 @@ namespace Duplicati.Server.Database
 

             return @default;

         }

-

                         

         private bool DeleteFromDb(string tablename, long id, System.Data.IDbTransaction transaction = null)                        

         {

diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs
index 645ec8cb3..b51041cca 100644
--- a/Duplicati/Server/Program.cs
+++ b/Duplicati/Server/Program.cs
@@ -400,6 +400,12 @@ namespace Duplicati.Server
                         Duplicati.Library.Utility.TempFile.RemoveOldApplicationTempFiles((path, ex) => {

                             Program.DataConnection.LogError(null, string.Format("Failed to delete temp file: {0}", path), ex); 

                         });

+

+                        string pts;

+                        if (!commandlineOptions.TryGetValue("log-retention", out pts))

+                            pts = DEFAULT_LOG_RETENTION;

+

+                        Program.DataConnection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true));

                     }

                     catch (Exception ex)

                     {

@@ -654,6 +660,11 @@ namespace Duplicati.Server
         }

 

         /// <summary>

+        /// The default log retention

+        /// </summary>

+        private static string DEFAULT_LOG_RETENTION = "30D";

+

+        /// <summary>

         /// Gets a list of all supported commandline options

         /// </summary>

         public static Library.Interface.ICommandLineArgument[] SupportedCommands

@@ -671,6 +682,7 @@ namespace Duplicati.Server
                     new Duplicati.Library.Interface.CommandLineArgument(Duplicati.Server.WebServer.Server.OPTION_INTERFACE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverInterfaceDescription, Strings.Program.WebserverInterfaceDescription, Duplicati.Server.WebServer.Server.DEFAULT_OPTION_INTERFACE),

                     new Duplicati.Library.Interface.CommandLineArgument("webservice-password", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Password, Strings.Program.WebserverPasswordDescription, Strings.Program.WebserverPasswordDescription),

                     new Duplicati.Library.Interface.CommandLineArgument("ping-pong-keepalive", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong),

+                    new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION),

                 };

             }

         }

diff --git a/Duplicati/Server/Strings.cs b/Duplicati/Server/Strings.cs
index fac429f3a..912f550c6 100644
--- a/Duplicati/Server/Strings.cs
+++ b/Duplicati/Server/Strings.cs
@@ -23,7 +23,8 @@ Error message: {0}", error); }
         public static string WebserverPasswordDescription { get { return LC.L(@"The password required to access the webserver. This option is saved so you do not need to set it on each run. Setting an empty value disables the password."); } }
         public static string PingpongkeepaliveShort { get { return LC.L(@"Enables the ping-pong responder"); } }
         public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read"); } }
-
+        public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } }
+        public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } }
     }
     internal static class TaskType {
         public static string FullBackup { get { return LC.L(@"Full backup"); } }