Implement exponential backoff for backend errors

5 files changed, 50 insertions, 3 deletions

diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs
index 904e300b1..97c80a2fd 100644
--- a/Duplicati/Library/Main/BackendManager.cs
+++ b/Duplicati/Library/Main/BackendManager.cs
@@ -363,6 +363,7 @@ namespace Duplicati.Library.Main
         // Cache these

         private readonly int m_numberofretries;

         private readonly TimeSpan m_retrydelay;

+        private readonly Boolean m_retrywithexponentialbackoff;

 

         public string BackendUrl { get { return m_backendurl; } }

 

@@ -374,6 +375,7 @@ namespace Duplicati.Library.Main
             m_taskControl = statwriter as BasicResults;

             m_numberofretries = options.NumberOfRetries;

             m_retrydelay = options.RetryDelay;

+            m_retrywithexponentialbackoff = options.RetryWithExponentialBackoff;

 

             m_db = new DatabaseCollector(database);

 

@@ -569,7 +571,9 @@ namespace Duplicati.Library.Main
 

                                 if (retries < m_numberofretries && m_retrydelay.Ticks != 0)

                                 {

-                                    var target = DateTime.Now.AddTicks(m_retrydelay.Ticks);

+                                    var delay = Library.Utility.Utility.GetRetryDelay(m_retrydelay, retries, m_retrywithexponentialbackoff);

+                                    var target = DateTime.Now.Add(delay);

+

                                     while (target > DateTime.Now)

                                     {

                                         if (m_taskControl != null && m_taskControl.IsAbortRequested())

diff --git a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs
index 307d60a52..7ff2ea107 100644
--- a/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs
+++ b/Duplicati/Library/Main/Operation/Backup/BackendUploader.cs
@@ -318,7 +318,10 @@ namespace Duplicati.Library.Main.Operation.Backup
             for (retryCount = 0; retryCount <= m_options.NumberOfRetries; retryCount++)

             {

                 if (m_options.RetryDelay.Ticks != 0 && retryCount != 0)

-                    await Task.Delay(m_options.RetryDelay).ConfigureAwait(false);

+                {

+                    var delay = Library.Utility.Utility.GetRetryDelay(m_options.RetryDelay, retryCount, m_options.RetryWithExponentialBackoff);

+                    await Task.Delay(delay).ConfigureAwait(false);

+                }

 

                 if (cancelToken.IsCancellationRequested)

                     return false;

diff --git a/Duplicati/Library/Main/Options.cs b/Duplicati/Library/Main/Options.cs
index d905f0865..a824f6680 100644
--- a/Duplicati/Library/Main/Options.cs
+++ b/Duplicati/Library/Main/Options.cs
@@ -280,6 +280,7 @@ namespace Duplicati.Library.Main
 

                     new CommandLineArgument("number-of-retries", CommandLineArgument.ArgumentType.Integer, Strings.Options.NumberofretriesShort, Strings.Options.NumberofretriesLong, "5"),

                     new CommandLineArgument("retry-delay", CommandLineArgument.ArgumentType.Timespan, Strings.Options.RetrydelayShort, Strings.Options.RetrydelayLong, "10s"),

+                    new CommandLineArgument("retry-with-exponential-backoff", CommandLineArgument.ArgumentType.Boolean, Strings.Options.RetrywithexponentialbackoffShort, Strings.Options.RetrywithexponentialbackoffLong, "false"),

 

                     new CommandLineArgument("synchronous-upload", CommandLineArgument.ArgumentType.Boolean, Strings.Options.SynchronousuploadShort, Strings.Options.SynchronousuploadLong, "false"),

                     new CommandLineArgument("asynchronous-upload-limit", CommandLineArgument.ArgumentType.Integer, Strings.Options.AsynchronousuploadlimitShort, Strings.Options.AsynchronousuploadlimitLong, "4"),

@@ -814,7 +815,7 @@ namespace Duplicati.Library.Main
         public bool DisablePipedStreaming { get { return GetBool("disable-piped-streaming"); } }

 

         /// <summary>

-        /// Gets the timelimit for removal

+        /// Gets the delay period to retry uploads

         /// </summary>

         public TimeSpan RetryDelay

         {

@@ -828,6 +829,14 @@ namespace Duplicati.Library.Main
         }

 

         /// <summary>

+        /// Gets whether exponential backoff is enabled

+        /// </summary>

+        public Boolean RetryWithExponentialBackoff

+        {

+            get { return Library.Utility.Utility.ParseBoolOption(m_options, "retry-with-exponential-backoff"); }

+        }

+

+        /// <summary>

         /// Gets the max upload speed in bytes pr. second

         /// </summary>

         public long MaxUploadPrSecond

diff --git a/Duplicati/Library/Main/Strings.cs b/Duplicati/Library/Main/Strings.cs
index 6208c40fe..9ba71054f 100644
--- a/Duplicati/Library/Main/Strings.cs
+++ b/Duplicati/Library/Main/Strings.cs
@@ -65,6 +65,8 @@ namespace Duplicati.Library.Main.Strings
         public static string ListfoldercontentsShort { get { return LC.L(@"Show folder contents"); } }

         public static string RetrydelayLong { get { return LC.L(@"After a failed transmission, Duplicati will wait a short period before attempting again. This is useful if the network drops out occasionally during transmissions."); } }

         public static string RetrydelayShort { get { return LC.L(@"Time to wait between retries"); } }

+        public static string RetrywithexponentialbackoffLong { get { return LC.L(@"After a failed transmission, Duplicati will wait a short period before attempting again. This period is controlled by the retry-delay option. When this option is enabled, that period is doubled after each consecutive failure."); } }

+        public static string RetrywithexponentialbackoffShort { get { return LC.L(@"Whether to enable exponential backoff."); } }

         public static string ControlfilesLong { get { return LC.L(@"Use this option to attach extra files to the newly uploaded filelists."); } }

         public static string ControlfilesShort { get { return LC.L(@"Set control files"); } }

         public static string SkipfilehashchecksLong { get { return LC.L(@"If the hash for the volume does not match, Duplicati will refuse to use the backup. Supply this flag to allow Duplicati to proceed anyway."); } }

diff --git a/Duplicati/Library/Utility/Utility.cs b/Duplicati/Library/Utility/Utility.cs
index ae6cc8a75..a41b2dc0f 100644
--- a/Duplicati/Library/Utility/Utility.cs
+++ b/Duplicati/Library/Utility/Utility.cs
@@ -1497,5 +1497,34 @@ namespace Duplicati.Library.Utility
         {

             return task.GetAwaiter().GetResult();

         }

+

+        /// <summary>

+        /// Utility that computes the delay before the next retry of an operation, optionally using exponential backoff.

+        /// Note: when using exponential backoff, the exponent is clamped at 10.

+        /// </summary>

+        /// <param name="retryDelay">Value of one delay unit</param>

+        /// <param name="retryAttempt">The attempt number (e.g. 1 for the first retry, 2 for the second retry, etc.)</param>

+        /// <param name="useExponentialBackoff">Whether to use exponential backoff</param>

+        /// <returns>The computed delay</returns>

+        public static TimeSpan GetRetryDelay(TimeSpan retryDelay, int retryAttempt, bool useExponentialBackoff)

+        {

+            if (retryAttempt < 1)

+            {

+                throw new ArgumentException("The attempt number must not be less than 1.", nameof(retryAttempt));

+            }

+

+            TimeSpan delay;

+            if (useExponentialBackoff)

+            {

+                var delayTicks = retryDelay.Ticks << Math.Min(retryAttempt - 1, 10);

+                delay = TimeSpan.FromTicks(delayTicks);

+            }

+            else

+            {

+                delay = retryDelay;

+            }

+

+            return delay;

+        }

     }

 }