Merge pull request #4145 from tygill/feature/ms-graph-non-httpclient

Allow MS Graph backends (e.g., OneDrive v2) to use OAuthHelper instead of OAuthHttpClient

6 files changed, 592 insertions, 138 deletions

diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs
index 8515841a3..a56c131fd 100644
--- a/Duplicati/CommandLine/BackendTester/Program.cs
+++ b/Duplicati/CommandLine/BackendTester/Program.cs
@@ -394,6 +394,30 @@ namespace Duplicati.CommandLine.BackendTester
                             Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name);

                         }

 

+                    // Test some error cases

+                    Console.WriteLine("Checking retrieval of non-existent file...");

+                    bool caughtExpectedException = false;

+                    try

+                    {

+                        using (Duplicati.Library.Utility.TempFile tempFile = new Duplicati.Library.Utility.TempFile())

+                        {

+                            backend.Get(string.Format("NonExistentFile-{0}", Guid.NewGuid()), tempFile.Name);

+                        }

+                    }

+                    catch (FileMissingException ex)

+                    {

+                        Console.WriteLine("Caught expected FileMissingException");

+                        caughtExpectedException = true;

+                    }

+                    catch (Exception ex)

+                    {

+                        Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex);

+                    }

+

+                    if (!caughtExpectedException)

+                    {

+                        Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException");

+                    }

                 }

 

                 // Test quota retrieval

diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs
index daeb7000e..a60a29028 100644
--- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs
+++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs
@@ -304,20 +304,48 @@ namespace Duplicati.Library
         {

         }

 

-        public HttpWebResponse GetResponseWithoutException(string url, string method = null)

+        public HttpWebResponse GetResponseWithoutException(string url, object requestdata = null, string method = null)

         {

-            return GetResponseWithoutException(CreateRequest(url, method));

+            if (requestdata is string)

+                throw new ArgumentException("Cannot send string object as data");

+

+            if (method == null && requestdata != null)

+                method = "POST";

+

+            return GetResponseWithoutException(CreateRequest(url, method), requestdata);

         }

 

-        public HttpWebResponse GetResponseWithoutException(HttpWebRequest req)

+        public HttpWebResponse GetResponseWithoutException(HttpWebRequest req, object requestdata = null)

         {

-            return GetResponseWithoutException(new AsyncHttpRequest(req));

+            return GetResponseWithoutException(new AsyncHttpRequest(req), requestdata);

         }

 

-        public HttpWebResponse GetResponseWithoutException(AsyncHttpRequest req)

+        public HttpWebResponse GetResponseWithoutException(AsyncHttpRequest req, object requestdata = null)

         {

             try

             {

+                if (requestdata != null)

+                {

+                    if (requestdata is Stream stream)

+                    {

+                        req.Request.ContentLength = stream.Length;

+                        if (string.IsNullOrEmpty(req.Request.ContentType))

+                            req.Request.ContentType = "application/octet-stream";

+

+                        using (var rs = req.GetRequestStream())

+                            Library.Utility.Utility.CopyStream(stream, rs);

+                    }

+                    else

+                    {

+                        var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata));

+                        req.Request.ContentLength = data.Length;

+                        req.Request.ContentType = "application/json; charset=UTF-8";

+

+                        using (var rs = req.GetRequestStream())

+                            rs.Write(data, 0, data.Length);

+                    }

+                }

+

                 return (HttpWebResponse)req.GetResponse();

             }

             catch(WebException wex)

@@ -329,6 +357,59 @@ namespace Duplicati.Library
             }

         }

 

+        public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(string url, CancellationToken cancelToken, object requestdata = null, string method = null)

+        {

+            if (requestdata is string)

+                throw new ArgumentException("Cannot send string object as data");

+

+            if (method == null && requestdata != null)

+                method = "POST";

+

+            return await GetResponseWithoutExceptionAsync(CreateRequest(url, method), cancelToken, requestdata).ConfigureAwait(false);

+        }

+

+        public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(HttpWebRequest req, CancellationToken cancelToken, object requestdata = null)

+        {

+            return await GetResponseWithoutExceptionAsync(new AsyncHttpRequest(req), cancelToken, requestdata).ConfigureAwait(false);

+        }

+

+        public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(AsyncHttpRequest req, CancellationToken cancelToken, object requestdata = null)

+        {

+            try

+            {

+                if (requestdata != null)

+                {

+                    if (requestdata is System.IO.Stream stream)

+                    {

+                        req.Request.ContentLength = stream.Length;

+                        if (string.IsNullOrEmpty(req.Request.ContentType))

+                            req.Request.ContentType = "application/octet-stream";

+

+                        using (var rs = req.GetRequestStream())

+                            await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken).ConfigureAwait(false);

+                    }

+                    else

+                    {

+                        var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata));

+                        req.Request.ContentLength = data.Length;

+                        req.Request.ContentType = "application/json; charset=UTF-8";

+

+                        using (var rs = req.GetRequestStream())

+                            await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);

+                    }

+                }

+

+                return (HttpWebResponse)req.GetResponse();

+            }

+            catch (WebException wex)

+            {

+                if (wex.Response is HttpWebResponse response)

+                    return response;

+

+                throw;

+            }

+        }

+

         public HttpWebResponse GetResponse(string url, object requestdata = null, string method = null)

         {

             if (requestdata is string)

@@ -405,7 +486,7 @@ namespace Duplicati.Library
                             req.Request.ContentType = "application/octet-stream";

 

                         using (var rs = req.GetRequestStream())

-                            await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken);

+                            await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken).ConfigureAwait(false);

                     }

                     else

                     {

@@ -414,7 +495,7 @@ namespace Duplicati.Library
                         req.Request.ContentType = "application/json; charset=UTF-8";

 

                         using (var rs = req.GetRequestStream())

-                            await rs.WriteAsync(data, 0, data.Length, cancelToken);

+                            await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);

                     }

                 }

 

diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs
index 1e971cc58..007376cc0 100644
--- a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs
+++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs
@@ -105,8 +105,13 @@ namespace Duplicati.Library
 

         public override HttpWebRequest CreateRequest(string url, string method = null)

         {

+            return this.CreateRequest(url, method, false);

+        }

+

+        public HttpWebRequest CreateRequest(string url, string method, bool noAuthorization)

+        {

             var r = base.CreateRequest(url, method);

-            if (AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url))

+            if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url))

                 r.Headers["Authorization"] = string.Format("Bearer {0}", AccessToken);

             return r;

         } 

diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs
index d31562da2..0eb50882d 100644
--- a/Duplicati/Library/Backend/OneDrive/Exceptions.cs
+++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs
@@ -1,102 +1,222 @@
-using System;
-using System.Net.Http;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-using Duplicati.Library.Utility;
-
-using Newtonsoft.Json;
-
-namespace Duplicati.Library.Backend.MicrosoftGraph
-{
-    public class MicrosoftGraphException : Exception
-    {
-        private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase);
-
-        public MicrosoftGraphException(HttpResponseMessage response)
-            : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response)
-        {
-        }
-
-        public MicrosoftGraphException(string message, HttpResponseMessage response)
-            : this(message, response, null)
-        {
-        }
-
-        public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException)
-            : base(BuildFullMessage(message, response), innerException)
-        {
-            this.Response = response;
-        }
-
-        public string RequestUrl => this.Response.RequestMessage.RequestUri.ToString();
-        public HttpResponseMessage Response { get; private set; }
-
-        protected static string ResponseToString(HttpResponseMessage response)
-        {
-            if (response != null)
-            {
-                // Start to read the content
-                Task<string> content = response.Content.ReadAsStringAsync();
-
-                // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.
-                // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.
-                // So we replace anything in the request that looks like the auth header with a safe version.
-                string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");
-                return string.Format("{0}\n{1}\n{2}", requestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content.Await()), Formatting.Indented));
-            }
-            else
-            {
-                return null;
-            }
-        }
-
-        private static string BuildFullMessage(string message, HttpResponseMessage response)
-        {
-            if (response != null)
-            {
-                return string.Format("{0}\n{1}", message, ResponseToString(response));
-            }
-            else
-            {
-                return message;
-            }
-        }
-    }
-
-    public class DriveItemNotFoundException : MicrosoftGraphException
-    {
-        public DriveItemNotFoundException(HttpResponseMessage response)
-            : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)
-        {
-        }
-    }
-
-    public class UploadSessionException : MicrosoftGraphException
-    {
-        public UploadSessionException(
-            HttpResponseMessage originalResponse,
-            int fragment,
-            int fragmentCount,
-            MicrosoftGraphException fragmentException)
-            : base(
-                  string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),
-                  originalResponse,
-                  fragmentException)
-        {
-            this.Fragment = fragment;
-            this.FragmentCount = fragmentCount;
-            this.InnerException = fragmentException;
-        }
-
-        public string CreateSessionRequestUrl => this.RequestUrl;
-        public HttpResponseMessage CreateSessionResponse => this.Response;
-
-        public int Fragment { get; private set; }
-        public int FragmentCount { get; private set; }
-        public string FragmentRequestUrl => this.InnerException.RequestUrl;
-        public HttpResponseMessage FragmentResponse => this.InnerException.Response;
-
-        public new MicrosoftGraphException InnerException { get; private set; }
-    }
-}
+using System;

+using System.IO;

+using System.Net;

+using System.Net.Http;

+using System.Text.RegularExpressions;

+using System.Threading.Tasks;

+

+using Duplicati.Library.Utility;

+

+using Newtonsoft.Json;

+

+namespace Duplicati.Library.Backend.MicrosoftGraph

+{

+    public class MicrosoftGraphException : Exception

+    {

+        private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase);

+

+        private readonly HttpResponseMessage responseMessage;

+        private readonly HttpWebResponse webResponse;

+

+        public MicrosoftGraphException(HttpResponseMessage response)

+            : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response)

+        {

+        }

+

+        public MicrosoftGraphException(string message, HttpResponseMessage response)

+            : this(message, response, null)

+        {

+        }

+

+        public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException)

+            : base(BuildFullMessage(message, response), innerException)

+        {

+            this.responseMessage = response;

+        }

+

+        public MicrosoftGraphException(HttpWebResponse response)

+            : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.StatusDescription, response.ResponseUri), response)

+        {

+        }

+

+        public MicrosoftGraphException(string message, HttpWebResponse response)

+            : this(message, response, null)

+        {

+        }

+

+        public MicrosoftGraphException(string message, HttpWebResponse response, Exception innerException)

+            : base(BuildFullMessage(message, response), innerException)

+        {

+            this.webResponse = response;

+        }

+

+        public string RequestUrl

+        {

+            get

+            {

+                if (this.responseMessage != null)

+                {

+                    return this.responseMessage.RequestMessage.RequestUri.ToString();

+                }

+                else

+                {

+                    return this.webResponse.ResponseUri.ToString();

+                }

+            }

+        }

+

+        public HttpStatusCode StatusCode

+        {

+            get

+            {

+                if (this.responseMessage != null)

+                {

+                    return this.responseMessage.StatusCode;

+                }

+                else

+                {

+                    return this.webResponse.StatusCode;

+                }

+            }

+        }

+

+        public HttpResponseMessage Response { get; private set; }

+

+        protected static string ResponseToString(HttpResponseMessage response)

+        {

+            if (response != null)

+            {

+                // Start to read the content

+                using (Task<string> content = response.Content.ReadAsStringAsync())

+                {

+                    // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.

+                    // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.

+                    // So we replace anything in the request that looks like the auth header with a safe version.

+                    string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");

+                    return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await()));

+                }

+            }

+            else

+            {

+                return null;

+            }

+        }

+

+        private static string BuildFullMessage(string message, HttpResponseMessage response)

+        {

+            if (response != null)

+            {

+                return string.Format("{0}\n{1}", message, ResponseToString(response));

+            }

+            else

+            {

+                return message;

+            }

+        }

+

+        protected static string ResponseToString(HttpWebResponse response)

+        {

+            if (response != null)

+            {

+                // Start to read the content

+                using (var responseStream = response.GetResponseStream())

+                using (var textReader = new StreamReader(responseStream))

+                {

+                    return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd()));

+                }

+            }

+            else

+            {

+                return null;

+            }

+        }

+

+        private static string BuildFullMessage(string message, HttpWebResponse response)

+        {

+            if (response != null)

+            {

+                return string.Format("{0}\n{1}", message, ResponseToString(response));

+            }

+            else

+            {

+                return message;

+            }

+        }

+

+        private static string PrettifyJson(string json)

+        {

+            if (string.IsNullOrWhiteSpace(json))

+            {

+                return json;

+            }

+

+            if (json[0] == '<')

+            {

+                // It looks like some errors return xml bodies instead of JSON.

+                // If this looks like it might be one of those, don't even bother parsing the JSON.

+                return json;

+            }

+

+            try

+            {

+                return JsonConvert.SerializeObject(JsonConvert.DeserializeObject(json), Formatting.Indented);

+            }

+            catch (Exception)

+            {

+                // Maybe this wasn't JSON..

+                return json;

+            }

+        }

+    }

+

+    public class DriveItemNotFoundException : MicrosoftGraphException

+    {

+        public DriveItemNotFoundException(HttpResponseMessage response)

+            : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)

+        {

+        }

+

+        public DriveItemNotFoundException(HttpWebResponse response)

+            : base(string.Format("Item at {0} was not found", response?.ResponseUri?.ToString() ?? "<unknown>"), response)

+        {

+        }

+    }

+

+    public class UploadSessionException : MicrosoftGraphException

+    {

+        public UploadSessionException(

+            HttpResponseMessage originalResponse,

+            int fragment,

+            int fragmentCount,

+            MicrosoftGraphException fragmentException)

+            : base(

+                  string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),

+                  originalResponse,

+                  fragmentException)

+        {

+            this.Fragment = fragment;

+            this.FragmentCount = fragmentCount;

+            this.InnerException = fragmentException;

+        }

+

+        public UploadSessionException(

+            HttpWebResponse originalResponse,

+            int fragment,

+            int fragmentCount,

+            MicrosoftGraphException fragmentException)

+            : base(

+                  string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.ResponseUri?.ToString() ?? "<unknown>"),

+                  originalResponse,

+                  fragmentException)

+        {

+            this.Fragment = fragment;

+            this.FragmentCount = fragmentCount;

+            this.InnerException = fragmentException;

+        }

+

+        public int Fragment { get; private set; }

+        public int FragmentCount { get; private set; }

+

+        public new MicrosoftGraphException InnerException { get; private set; }

+    }

+}

diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs
index 1983d43e1..c8e533fc7 100644
--- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs
+++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs
@@ -39,6 +39,7 @@ namespace Duplicati.Library.Backend
         private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size";

         private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count";

         private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay";

+        private const string USE_HTTP_CLIENT = "use-http-client";

 

         private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT = 5;

         private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 1000;

@@ -68,12 +69,27 @@ namespace Duplicati.Library.Backend
         /// </summary>

         private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024;

 

+        /// <summary>

+        /// Whether to use the HttpClient class for HTTP requests.

+        /// Default is false when running under Mono (as it seems it might be causing a memory leak in some environments / versions)

+        /// but true in other cases (where these memory leaks haven't been reproduced).

+        /// </summary>

+        private static readonly bool USE_HTTP_CLIENT_DEFAULT = Utility.Utility.IsMono ? false : true;

+

         private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH");

 

+        /// <summary>

+        /// Dummy UploadSession given as an empty body to createUploadSession requests when using the OAuthHelper instead of the OAuthHttpClient.

+        /// The API expects a ContentLength to be specified, but the body content is optional.

+        /// Passing an empty object (or specifying the ContentLength explicitly) bypasses this error.

+        /// </summary>

+        private static readonly UploadSession dummyUploadSession = new UploadSession();

+

         protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink);

 

         private readonly JsonSerializer m_serializer = new JsonSerializer();

         private readonly OAuthHttpClient m_client;

+        private readonly OAuthHelper m_oAuthHelper;

         private readonly int fragmentSize;

         private readonly int fragmentRetryCount;

         private readonly int fragmentRetryDelay; // In milliseconds

@@ -119,8 +135,27 @@ namespace Duplicati.Library.Backend
                 this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY;

             }

 

-            this.m_client = new OAuthHttpClient(authid, protocolKey);

-            this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);

+            bool useHttpClient;

+            string useHttpClientStr;

+            if (options.TryGetValue(USE_HTTP_CLIENT, out useHttpClientStr))

+            {

+                useHttpClient = Utility.Utility.ParseBool(useHttpClientStr, USE_HTTP_CLIENT_DEFAULT);

+            }

+            else

+            {

+                useHttpClient = USE_HTTP_CLIENT_DEFAULT;

+            }

+

+            if (useHttpClient)

+            {

+                this.m_client = new OAuthHttpClient(authid, protocolKey);

+                this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);

+            }

+            else

+            {

+                this.m_oAuthHelper = new OAuthHelper(authid, protocolKey);

+                this.m_oAuthHelper.AutoAuthHeader = true;

+            }

 

             // Extract out the path to the backup root folder from the given URI.  Since this can be an expensive operation, 

             // we will cache the value using a lazy initializer.

@@ -153,6 +188,7 @@ namespace Duplicati.Library.Backend
                     new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)),

                     new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()),

                     new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()),

+                    new CommandLineArgument(USE_HTTP_CLIENT, CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, USE_HTTP_CLIENT_DEFAULT.ToString()),

                 }

                 .Concat(this.AdditionalSupportedCommands).ToList();

             }

@@ -170,13 +206,23 @@ namespace Duplicati.Library.Backend
                     // To get the upload session endpoint, we can start an upload session and then immediately cancel it.

                     // We pick a random file name (using a guid) to make sure we don't conflict with an existing file

                     string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());

-                    UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), null);

+                    UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), MicrosoftGraphBackend.dummyUploadSession);

 

                     // Canceling an upload session is done by sending a DELETE to the upload URL

-                    using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl))

-                    using (var response = this.m_client.SendAsync(request).Await())

+                    if (this.m_client != null)

                     {

-                        this.CheckResponse(response);

+                        using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl))

+                        using (var response = this.m_client.SendAsync(request).Await())

+                        {

+                            this.CheckResponse(response);

+                        }

+                    }

+                    else

+                    {

+                        using (var response = this.m_oAuthHelper.GetResponseWithoutException(uploadSession.UploadUrl, MicrosoftGraphBackend.dummyUploadSession, HttpMethod.Delete.ToString()))

+                        {

+                            this.CheckResponse(response);

+                        }

                     }

 

                     this.dnsNames = new[]

@@ -238,7 +284,18 @@ namespace Duplicati.Library.Backend
 

         private string DrivePrefix

         {

-            get { return this.ApiVersion + this.DrivePath; }

+            get

+            {

+                if (this.m_client != null)

+                {

+                    return this.ApiVersion + this.DrivePath;

+                }

+                else

+                {

+                    // When not using the HttpClient for requests, the base address needs to be included in this prefix

+                    return BASE_ADDRESS + this.ApiVersion + this.DrivePath;

+                }

+            }

         }

 

         public void CreateFolder()

@@ -297,12 +354,27 @@ namespace Duplicati.Library.Backend
         {

             try

             {

-                using (var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await())

+                string getUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));

+                if (this.m_client != null)

                 {

-                    this.CheckResponse(response);

-                    using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())

+                    using (var response = this.m_client.GetAsync(getUrl).Await())

                     {

-                        responseStream.CopyTo(stream);

+                        this.CheckResponse(response);

+                        using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())

+                        {

+                            responseStream.CopyTo(stream);

+                        }

+                    }

+                }

+                else

+                {

+                    using (var response = this.m_oAuthHelper.GetResponseWithoutException(getUrl))

+                    {

+                        this.CheckResponse(response);

+                        using (Stream responseStream = response.GetResponseStream())

+                        {

+                            responseStream.CopyTo(stream);

+                        }

                     }

                 }

             }

@@ -339,10 +411,22 @@ namespace Duplicati.Library.Backend
             // PUT only supports up to 4 MB file uploads. There's a separate process for larger files.

             if (stream.Length < PUT_MAX_SIZE)

             {

-                using (StreamContent streamContent = new StreamContent(stream))

+                string putUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));

+                if (this.m_client != null)

+                {

+                    using (StreamContent streamContent = new StreamContent(stream))

+                    {

+                        streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");

+                        using (var response = await this.m_client.PutAsync(putUrl, streamContent, cancelToken).ConfigureAwait(false))

+                        {

+                            // Make sure this response is a valid drive item, though we don't actually use it for anything currently.

+                            this.ParseResponse<DriveItem>(response);

+                        }

+                    }

+                }

+                else

                 {

-                    streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");

-                    using (var response = await m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)), streamContent, cancelToken).ConfigureAwait(false))

+                    using (var response = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(putUrl, cancelToken, stream, HttpMethod.Put.ToString()).ConfigureAwait(false))

                     {

                         // Make sure this response is a valid drive item, though we don't actually use it for anything currently.

                         this.ParseResponse<DriveItem>(response);

@@ -356,9 +440,11 @@ namespace Duplicati.Library.Backend
                 // The documentation seems somewhat contradictory - it states that uploads must be done sequentially,

                 // but also states that the nextExpectedRanges value returned may indicate multiple ranges...

                 // For now, this plays it safe and does a sequential upload.

-                using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))))

+                string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));

+                if (this.m_client != null)

                 {

-                    using (HttpResponseMessage createSessionResponse = await m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false))

+                    using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl))

+                    using (HttpResponseMessage createSessionResponse = await this.m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false))

                     {

                         UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);

 

@@ -385,7 +471,7 @@ namespace Duplicati.Library.Backend
                                     try

                                     {

                                         // The uploaded put requests will error if they are authenticated

-                                        using (HttpResponseMessage response = await m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))

+                                        using (HttpResponseMessage response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))

                                         {

                                             // Note: On the last request, the json result includes the default properties of the item that was uploaded

                                             this.ParseResponse<UploadSession>(response);

@@ -400,20 +486,20 @@ namespace Duplicati.Library.Backend
                                             // We've used up all our retry attempts

                                             throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);

                                         }

-                                        else if ((int)ex.Response.StatusCode >= 500 && (int)ex.Response.StatusCode < 600)

+                                        else if ((int)ex.StatusCode >= 500 && (int)ex.StatusCode < 600)

                                         {

                                             // If a 5xx error code is hit, we should use an exponential backoff strategy before retrying.

                                             // To make things simpler, we just use the current attempt number as the exponential factor.

                                             Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await()

                                             continue;

                                         }

-                                        else if (ex.Response.StatusCode == HttpStatusCode.NotFound)

+                                        else if (ex.StatusCode == HttpStatusCode.NotFound)

                                         {

                                             // 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.

                                             // Instead we'll let the caller re-attempt the whole file.

                                             throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);

                                         }

-                                        else if ((int)ex.Response.StatusCode >= 400 && (int)ex.Response.StatusCode < 500)

+                                        else if ((int)ex.StatusCode >= 400 && (int)ex.StatusCode < 500)

                                         {

                                             // If a 4xx error code is hit, we should retry without the backoff attempt

                                             continue;

@@ -432,6 +518,83 @@ namespace Duplicati.Library.Backend
                         }

                     }

                 }

+                else

+                {

+                    using (HttpWebResponse createSessionResponse = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(createSessionUrl, cancelToken, MicrosoftGraphBackend.dummyUploadSession, HttpMethod.Post.ToString()))

+                    {

+                        UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);

+

+                        // If the stream's total length is less than the chosen fragment size, then we should make the buffer only as large as the stream.

+                        int bufferSize = (int)Math.Min(this.fragmentSize, stream.Length);

+

+                        byte[] fragmentBuffer = new byte[bufferSize];

+                        int read = 0;

+                        for (int offset = 0; offset < stream.Length; offset += read)

+                        {

+                            read = await stream.ReadAsync(fragmentBuffer, 0, bufferSize, cancelToken).ConfigureAwait(false);

+

+                            int retryCount = this.fragmentRetryCount;

+                            for (int attempt = 0; attempt < retryCount; attempt++)

+                            {

+                                // The uploaded put requests will error if they are authenticated

+                                var request = new AsyncHttpRequest(this.m_oAuthHelper.CreateRequest(uploadSession.UploadUrl, HttpMethod.Put.ToString(), true));

+                                request.Request.ContentLength = read;

+                                request.Request.Headers.Set(HttpRequestHeader.ContentRange, new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length).ToString());

+                                request.Request.ContentType = "application/octet-stream";

+

+                                using (var requestStream = request.GetRequestStream(read))

+                                {

+                                    await requestStream.WriteAsync(fragmentBuffer, 0, read, cancelToken);

+                                }

+

+                                try

+                                {

+                                    using (var response = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(request, cancelToken))

+                                    {

+                                        // Note: On the last request, the json result includes the default properties of the item that was uploaded

+                                        this.ParseResponse<UploadSession>(response);

+                                    }

+                                }

+                                catch (MicrosoftGraphException ex)

+                                {

+                                    // Error handling based on recommendations here:

+                                    // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession#best-practices

+                                    if (attempt >= retryCount - 1)

+                                    {

+                                        // We've used up all our retry attempts

+                                        throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);

+                                    }

+                                    else if ((int)ex.StatusCode >= 500 && (int)ex.StatusCode < 600)

+                                    {

+                                        // If a 5xx error code is hit, we should use an exponential backoff strategy before retrying.

+                                        // To make things simpler, we just use the current attempt number as the exponential factor.

+                                        Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await()

+                                        continue;

+                                    }

+                                    else if (ex.StatusCode == HttpStatusCode.NotFound)

+                                    {

+                                        // 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.

+                                        // Instead we'll let the caller re-attempt the whole file.

+                                        throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);

+                                    }

+                                    else if ((int)ex.StatusCode >= 400 && (int)ex.StatusCode < 500)

+                                    {

+                                        // If a 4xx error code is hit, we should retry without the backoff attempt

+                                        continue;

+                                    }

+                                    else

+                                    {

+                                        // Other errors should be rethrown

+                                        throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);

+                                    }

+                                }

+

+                                // If we successfully sent this piece, then we can break out of the retry loop

+                                break;

+                            }

+                        }

+                    }

+                }

             }

         }

 

@@ -439,9 +602,20 @@ namespace Duplicati.Library.Backend
         {

             try

             {

-                using (var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await())

+                string deleteUrl = string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));

+                if (this.m_client != null)

+                {

+                    using (var response = this.m_client.DeleteAsync(deleteUrl).Await())

+                    {

+                        this.CheckResponse(response);

+                    }

+                }

+                else

                 {

-                    this.CheckResponse(response);

+                    using (var response = this.m_oAuthHelper.GetResponseWithoutException(deleteUrl, null, HttpMethod.Delete.ToString()))

+                    {

+                        this.CheckResponse(response);

+                    }

                 }

             }

             catch (DriveItemNotFoundException ex)

@@ -498,18 +672,38 @@ namespace Duplicati.Library.Backend
 

         private T SendRequest<T>(HttpMethod method, string url)

         {

-            using (var request = new HttpRequestMessage(method, url))

+            if (this.m_client != null)

             {

-                return this.SendRequest<T>(request);

+                using (var request = new HttpRequestMessage(method, url))

+                {

+                    return this.SendRequest<T>(request);

+                }

+            }

+            else

+            {

+                using (var response = this.m_oAuthHelper.GetResponseWithoutException(url, null, method.ToString()))

+                {

+                    return this.ParseResponse<T>(response);

+                }

             }

         }

 

         private T SendRequest<T>(HttpMethod method, string url, T body) where T : class

         {

-            using (var request = new HttpRequestMessage(method, url))

-            using (request.Content = this.PrepareContent(body))

+            if (this.m_client != null)

             {

-                return this.SendRequest<T>(request);

+                using (var request = new HttpRequestMessage(method, url))

+                using (request.Content = this.PrepareContent(body))

+                {

+                    return this.SendRequest<T>(request);

+                }

+            }

+            else

+            {

+                using (var response = this.m_oAuthHelper.GetResponseWithoutException(url, body, method.ToString()))

+                {

+                    return this.ParseResponse<T>(response);

+                }

             }

         }

 

@@ -520,7 +714,7 @@ namespace Duplicati.Library.Backend
                 return this.ParseResponse<T>(response);

             }

         }

-

+        

         private IEnumerable<T> Enumerate<T>(string url)

         {

             string nextUrl = url;

@@ -563,6 +757,23 @@ namespace Duplicati.Library.Backend
             }

         }

 

+        private void CheckResponse(HttpWebResponse response)

+        {

+            if (!((int)response.StatusCode >= 200 && (int)response.StatusCode < 300))

+            {

+                if (response.StatusCode == HttpStatusCode.NotFound)

+                {

+                    // It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out.

+                    throw new DriveItemNotFoundException(response);

+                }

+                else

+                {

+                    // Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc.

+                    throw new MicrosoftGraphException(response);

+                }

+            }

+        }

+

         private T ParseResponse<T>(HttpResponseMessage response)

         {

             this.CheckResponse(response);

@@ -574,6 +785,17 @@ namespace Duplicati.Library.Backend
             }

         }

 

+        private T ParseResponse<T>(HttpWebResponse response)

+        {

+            this.CheckResponse(response);

+            using (Stream responseStream = response.GetResponseStream())

+            using (StreamReader reader = new StreamReader(responseStream))

+            using (JsonTextReader jsonReader = new JsonTextReader(reader))

+            {

+                return this.m_serializer.Deserialize<T>(jsonReader);

+            }

+        }

+

         /// <summary>

         /// Normalizes the slashes in a url fragment. For example:

         ///   "" => ""

diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs
index a1b0a05c5..d2892d0c9 100644
--- a/Duplicati/Library/Backend/OneDrive/Strings.cs
+++ b/Duplicati/Library/Backend/OneDrive/Strings.cs
@@ -12,6 +12,8 @@ namespace Duplicati.Library.Backend.Strings
         public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload"); } }

         public static string FragmentRetryDelayShort { get { return LC.L(@"Millisecond delay between fragment errors"); } }

         public static string FragmentRetryDelayLong { get { return LC.L(@"Amount of time (in milliseconds) to wait between failures when uploading fragments"); } }

+        public static string UseHttpClientShort { get { return LC.L(@"Whether the HttpClient class should be used"); } }

+        public static string UseHttpClientLong { get { return LC.L(@"Whether the HttpClient class should be used to perform HTTP requests"); } }

     }

 

     internal static class OneDriveV2