From 397221175799486d451de05b630a2fe0179fc033 Mon Sep 17 00:00:00 2001 From: Kenneth Skovhede Date: Sun, 15 Feb 2015 23:26:52 +0100 Subject: Added automatic log cleanup This fixes #1298 --- Duplicati/Library/Main/Database/LocalDatabase.cs | 15 ++++++++++++++ Duplicati/Library/Main/Operation/BackupHandler.cs | 1 + Duplicati/Library/Main/Options.cs | 23 +++++++++++++++++++++ Duplicati/Library/Main/Strings.cs | 2 ++ Duplicati/Server/Database/Connection.cs | 25 ++++++++++++++++++++++- Duplicati/Server/Program.cs | 12 +++++++++++ Duplicati/Server/Strings.cs | 3 ++- 7 files changed, 79 insertions(+), 2 deletions(-) diff --git a/Duplicati/Library/Main/Database/LocalDatabase.cs b/Duplicati/Library/Main/Database/LocalDatabase.cs index 2149d9270..b8cfbbde8 100644 --- a/Duplicati/Library/Main/Database/LocalDatabase.cs +++ b/Duplicati/Library/Main/Database/LocalDatabase.cs @@ -934,6 +934,21 @@ namespace Duplicati.Library.Main.Database } } + + public void PurgeLogData(DateTime threshold) + { + using(var tr = m_connection.BeginTransaction()) + using(var cmd = m_connection.CreateCommand(tr)) + { + var t = NormalizeDateTimeToEpochSeconds(threshold); + cmd.ExecuteNonQuery(@"DELETE FROM ""LogData"" WHERE ""Timestamp"" < ?", t); + cmd.ExecuteNonQuery(@"DELETE FROM ""RemoteOperation"" WHERE ""Timestamp"" < ?", t); + + tr.Commit(); + } + using(var cmd = m_connection.CreateCommand()) + cmd.ExecuteNonQuery("VACUUM"); + } public virtual void Dispose() { diff --git a/Duplicati/Library/Main/Operation/BackupHandler.cs b/Duplicati/Library/Main/Operation/BackupHandler.cs index e9d622427..1bd24293c 100644 --- a/Duplicati/Library/Main/Operation/BackupHandler.cs +++ b/Duplicati/Library/Main/Operation/BackupHandler.cs @@ -738,6 +738,7 @@ namespace Duplicati.Library.Main.Operation m_result.OperationProgressUpdater.UpdatePhase(OperationPhase.Backup_Complete); m_database.WriteResults(); + m_database.PurgeLogData(m_options.LogRetention); return; } } diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index 10703a50a..f8a9a3d97 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -68,6 +68,11 @@ namespace Duplicati.Library.Main /// private const int DEFAULT_KEEP_VERSIONS = 0; + /// + /// The default threshold for purging log data + /// + private const string DEFAULT_LOG_RETENTION = "30D"; + /// /// An enumeration that describes the supported strategies for an optimization /// @@ -499,6 +504,9 @@ namespace Duplicati.Library.Main new CommandLineArgument("allow-passphrase-change", CommandLineArgument.ArgumentType.Boolean, Strings.Options.AllowpassphrasechangeShort, Strings.Options.AllowpassphrasechangeLong, "false"), new CommandLineArgument("no-local-blocks", CommandLineArgument.ArgumentType.Boolean, Strings.Options.NolocalblocksShort, Strings.Options.NolocalblocksLong, "false"), new CommandLineArgument("full-block-verification", CommandLineArgument.ArgumentType.Boolean, Strings.Options.FullblockverificationShort, Strings.Options.FullblockverificationLong, "false"), + + new CommandLineArgument("log-retention", CommandLineArgument.ArgumentType.Timespan, Strings.Options.LogretentionShort, Strings.Options.LogretentionLong, DEFAULT_LOG_RETENTION), + }); return lst; @@ -1719,6 +1727,21 @@ namespace Duplicati.Library.Main get { return Library.Utility.Utility.ParseBoolOption(m_options, "full-block-verification"); } } + + /// + /// Gets the threshold for when log data should be cleaned + /// + public DateTime LogRetention + { + get + { + string pts; + if (!m_options.TryGetValue("log-retention", out pts)) + pts = DEFAULT_LOG_RETENTION; + + return Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true); + } + } /// /// Gets a lookup table with compression hints, the key is the file extension with the leading period /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index f3ef7db21..68c3f4410 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -199,6 +199,8 @@ namespace Duplicati.Library.Main.Strings public static string NolocalblocksLong { get { return LC.L(@"Duplicati will attempt to use data from source files to minimize the amount of downloaded data. Use this option to skip this optimization and only use remote data."); } } public static string FullblockverificationShort { get { return LC.L(@"Check block hashes"); } } public static string FullblockverificationLong { get { return LC.L(@"Use this option to increase verification by checking the hash of blocks read from a volume before patching restored files with the data."); } } + public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } } + public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } } } internal static class Foresthash diff --git a/Duplicati/Server/Database/Connection.cs b/Duplicati/Server/Database/Connection.cs index 88ec66cb3..34a28fd5a 100644 --- a/Duplicati/Server/Database/Connection.cs +++ b/Duplicati/Server/Database/Connection.cs @@ -786,6 +786,30 @@ namespace Duplicati.Server.Database ticks -= ticks % TimeSpan.TicksPerSecond; return new DateTime(ticks, DateTimeKind.Utc); } + + public void PurgeLogData(DateTime purgeDate) + { + var t = NormalizeDateTimeToEpochSeconds(purgeDate); + + using(var tr = m_connection.BeginTransaction()) + using(var cmd = m_connection.CreateCommand()) + { + cmd.Transaction = tr; + cmd.CommandText = @"DELETE FROM ""ErrorLog"" WHERE ""Timestamp"" < ?"; + cmd.Parameters.Add(cmd.CreateParameter()); + ((System.Data.IDataParameter)cmd.Parameters[0]).Value = t; + cmd.ExecuteNonQuery(); + + tr.Commit(); + } + + using(var cmd = m_connection.CreateCommand()) + { + cmd.CommandText = "VACUUM"; + cmd.ExecuteNonQuery(); + } + + } private static long NormalizeDateTimeToEpochSeconds(DateTime input) { @@ -869,7 +893,6 @@ namespace Duplicati.Server.Database return @default; } - private bool DeleteFromDb(string tablename, long id, System.Data.IDbTransaction transaction = null) { diff --git a/Duplicati/Server/Program.cs b/Duplicati/Server/Program.cs index 645ec8cb3..b51041cca 100644 --- a/Duplicati/Server/Program.cs +++ b/Duplicati/Server/Program.cs @@ -400,6 +400,12 @@ namespace Duplicati.Server Duplicati.Library.Utility.TempFile.RemoveOldApplicationTempFiles((path, ex) => { Program.DataConnection.LogError(null, string.Format("Failed to delete temp file: {0}", path), ex); }); + + string pts; + if (!commandlineOptions.TryGetValue("log-retention", out pts)) + pts = DEFAULT_LOG_RETENTION; + + Program.DataConnection.PurgeLogData(Library.Utility.Timeparser.ParseTimeInterval(pts, DateTime.Now, true)); } catch (Exception ex) { @@ -653,6 +659,11 @@ namespace Duplicati.Server } } + /// + /// The default log retention + /// + private static string DEFAULT_LOG_RETENTION = "30D"; + /// /// Gets a list of all supported commandline options /// @@ -671,6 +682,7 @@ namespace Duplicati.Server new Duplicati.Library.Interface.CommandLineArgument(Duplicati.Server.WebServer.Server.OPTION_INTERFACE, Duplicati.Library.Interface.CommandLineArgument.ArgumentType.String, Strings.Program.WebserverInterfaceDescription, Strings.Program.WebserverInterfaceDescription, Duplicati.Server.WebServer.Server.DEFAULT_OPTION_INTERFACE), new Duplicati.Library.Interface.CommandLineArgument("webservice-password", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Password, Strings.Program.WebserverPasswordDescription, Strings.Program.WebserverPasswordDescription), new Duplicati.Library.Interface.CommandLineArgument("ping-pong-keepalive", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Boolean, Strings.Program.PingpongkeepaliveShort, Strings.Program.PingpongkeepaliveLong), + new Duplicati.Library.Interface.CommandLineArgument("log-retention", Duplicati.Library.Interface.CommandLineArgument.ArgumentType.Timespan, Strings.Program.LogretentionShort, Strings.Program.LogretentionLong, DEFAULT_LOG_RETENTION), }; } } diff --git a/Duplicati/Server/Strings.cs b/Duplicati/Server/Strings.cs index fac429f3a..912f550c6 100644 --- a/Duplicati/Server/Strings.cs +++ b/Duplicati/Server/Strings.cs @@ -23,7 +23,8 @@ Error message: {0}", error); } public static string WebserverPasswordDescription { get { return LC.L(@"The password required to access the webserver. This option is saved so you do not need to set it on each run. Setting an empty value disables the password."); } } public static string PingpongkeepaliveShort { get { return LC.L(@"Enables the ping-pong responder"); } } public static string PingpongkeepaliveLong { get { return LC.L(@"When running as a server, the service daemon must verify that the process is responding. If this option is enabled, the server reads stdin and writes a reply to each line read"); } } - + public static string LogretentionShort { get { return LC.L(@"Clean up old log data"); } } + public static string LogretentionLong { get { return LC.L(@"Set the time after which log data will be purged from the database."); } } } internal static class TaskType { public static string FullBackup { get { return LC.L(@"Full backup"); } } -- cgit v1.2.3