From 528478fda480da09deb7db3cf74d5f027224c843 Mon Sep 17 00:00:00 2001 From: Riccardo Paolo Bestetti Date: Sat, 15 Jan 2022 18:11:59 +0100 Subject: Implement exponential backoff for backend errors --- Duplicati/Library/Main/BackendManager.cs | 6 ++++- .../Main/Operation/Backup/BackendUploader.cs | 5 +++- Duplicati/Library/Main/Options.cs | 11 +++++++- Duplicati/Library/Main/Strings.cs | 2 ++ Duplicati/Library/Utility/Utility.cs | 29 ++++++++++++++++++++++ 5 files changed, 50 insertions(+), 3 deletions(-) (limited to 'Duplicati/Library') diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs index 904e300b1..97c80a2fd 100644 --- a/Duplicati/Library/Main/BackendManager.cs +++ b/Duplicati/Library/Main/BackendManager.cs @@ -363,6 +363,7 @@ namespace Duplicati.Library.Main // Cache these private readonly int m_numberofretries; private readonly TimeSpan m_retrydelay; + private readonly Boolean m_retrywithexponentialbackoff; public string BackendUrl { get { return m_backendurl; } } @@ -374,6 +375,7 @@ namespace Duplicati.Library.Main m_taskControl = statwriter as BasicResults; m_numberofretries = options.NumberOfRetries; m_retrydelay = options.RetryDelay; + m_retrywithexponentialbackoff = options.RetryWithExponentialBackoff; m_db = new DatabaseCollector(database); @@ -569,7 +571,9 @@ namespace Duplicati.Library.Main if (retries < m_numberofretries && m_retrydelay.Ticks != 0) { - var target = DateTime.Now.AddTicks(m_retrydelay.Ticks); + var delay = Library.Utility.Utility.GetRetryDelay(m_retrydelay, retries, m_retrywithexponentialbackoff); + var target = DateTime.Now.Add(delay); + while (target > DateTime.Now) { if (m_taskControl != null && m_taskControl.IsAbortRequested()) diff --git a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs index 307d60a52..7ff2ea107 100644 --- a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs +++ b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs @@ -318,7 +318,10 @@ namespace Duplicati.Library.Main.Operation.Backup for (retryCount = 0; retryCount <= m_options.NumberOfRetries; retryCount++) { if (m_options.RetryDelay.Ticks != 0 && retryCount != 0) - await Task.Delay(m_options.RetryDelay).ConfigureAwait(false); + { + var delay = Library.Utility.Utility.GetRetryDelay(m_options.RetryDelay, retryCount, m_options.RetryWithExponentialBackoff); + await Task.Delay(delay).ConfigureAwait(false); + } if (cancelToken.IsCancellationRequested) return false; diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs index d905f0865..a824f6680 100644 --- a/Duplicati/Library/Main/Options.cs +++ b/Duplicati/Library/Main/Options.cs @@ -280,6 +280,7 @@ namespace Duplicati.Library.Main new CommandLineArgument("number-of-retries", CommandLineArgument.ArgumentType.Integer, Strings.Options.NumberofretriesShort, Strings.Options.NumberofretriesLong, "5"), new CommandLineArgument("retry-delay", CommandLineArgument.ArgumentType.Timespan, Strings.Options.RetrydelayShort, Strings.Options.RetrydelayLong, "10s"), + new CommandLineArgument("retry-with-exponential-backoff", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RetrywithexponentialbackoffShort, Strings.Options.RetrywithexponentialbackoffLong, "false"), new CommandLineArgument("synchronous-upload", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SynchronousuploadShort, Strings.Options.SynchronousuploadLong, "false"), new CommandLineArgument("asynchronous-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousuploadlimitShort, Strings.Options.AsynchronousuploadlimitLong, "4"), @@ -814,7 +815,7 @@ namespace Duplicati.Library.Main public bool DisablePipedStreaming { get { return GetBool("disable-piped-streaming"); } } /// - /// Gets the timelimit for removal + /// Gets the delay period to retry uploads /// public TimeSpan RetryDelay { @@ -827,6 +828,14 @@ namespace Duplicati.Library.Main } } + /// + /// Gets whether exponential backoff is enabled + /// + public Boolean RetryWithExponentialBackoff + { + get { return Library.Utility.Utility.ParseBoolOption(m_options, "retry-with-exponential-backoff"); } + } + /// /// Gets the max upload speed in bytes pr. second /// diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs index 6208c40fe..9ba71054f 100644 --- a/Duplicati/Library/Main/Strings.cs +++ b/Duplicati/Library/Main/Strings.cs @@ -65,6 +65,8 @@ namespace Duplicati.Library.Main.Strings public static string ListfoldercontentsShort { get { return LC.L(@"Show folder contents"); } } public static string RetrydelayLong { get { return LC.L(@"After a failed transmission, Duplicati will wait a short period before attempting again. This is useful if the network drops out occasionally during transmissions."); } } public static string RetrydelayShort { get { return LC.L(@"Time to wait between retries"); } } + public static string RetrywithexponentialbackoffLong { get { return LC.L(@"After a failed transmission, Duplicati will wait a short period before attempting again. This period is controlled by the retry-delay option. When this option is enabled, that period is doubled after each consecutive failure."); } } + public static string RetrywithexponentialbackoffShort { get { return LC.L(@"Whether to enable exponential backoff."); } } public static string ControlfilesLong { get { return LC.L(@"Use this option to attach extra files to the newly uploaded filelists."); } } public static string ControlfilesShort { get { return LC.L(@"Set control files"); } } public static string SkipfilehashchecksLong { get { return LC.L(@"If the hash for the volume does not match, Duplicati will refuse to use the backup. Supply this flag to allow Duplicati to proceed anyway."); } } diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs index ae6cc8a75..a41b2dc0f 100644 --- a/Duplicati/Library/Utility/Utility.cs +++ b/Duplicati/Library/Utility/Utility.cs @@ -1497,5 +1497,34 @@ namespace Duplicati.Library.Utility { return task.GetAwaiter().GetResult(); } + + /// + /// Utility that computes the delay before the next retry of an operation, optionally using exponential backoff. + /// Note: when using exponential backoff, the exponent is clamped at 10. + /// + /// Value of one delay unit + /// The attempt number (e.g. 1 for the first retry, 2 for the second retry, etc.) + /// Whether to use exponential backoff + /// The computed delay + public static TimeSpan GetRetryDelay(TimeSpan retryDelay, int retryAttempt, bool useExponentialBackoff) + { + if (retryAttempt < 1) + { + throw new ArgumentException("The attempt number must not be less than 1.", nameof(retryAttempt)); + } + + TimeSpan delay; + if (useExponentialBackoff) + { + var delayTicks = retryDelay.Ticks << Math.Min(retryAttempt - 1, 10); + delay = TimeSpan.FromTicks(delayTicks); + } + else + { + delay = retryDelay; + } + + return delay; + } } } -- cgit v1.2.3