diff options
author | Kenneth Skovhede <kenneth@hexad.dk> | 2020-03-25 09:09:44 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-03-25 09:09:44 +0300 |
commit | 6a8912c4736bbe8bd9e61be561f339e3f16f48ab (patch) | |
tree | 5f6d92e9ddb13e7f6ab8341e1d51eef119fda966 | |
parent | 3a08c8ca2d797068aca5a751615f7756a5939b9b (diff) | |
parent | 1ae534e45e6bb245abb1470a8580292ca24a818e (diff) |
Merge pull request #4145 from tygill/feature/ms-graph-non-httpclient
Allow MS Graph backends (e.g., OneDrive v2) to use OAuthHelper instead of OAuthHttpClient
-rw-r--r-- | Duplicati/CommandLine/BackendTester/Program.cs | 24 | ||||
-rw-r--r-- | Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs | 95 | ||||
-rw-r--r-- | Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs | 7 | ||||
-rw-r--r-- | Duplicati/Library/Backend/OneDrive/Exceptions.cs | 324 | ||||
-rw-r--r-- | Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs | 278 | ||||
-rw-r--r-- | Duplicati/Library/Backend/OneDrive/Strings.cs | 2 |
6 files changed, 592 insertions, 138 deletions
diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 8515841a3..a56c131fd 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -394,6 +394,30 @@ namespace Duplicati.CommandLine.BackendTester Console.WriteLine("*** Remote folder contains {0} after cleanup", fe.Name);
}
+ // Test some error cases
+ Console.WriteLine("Checking retrieval of non-existent file...");
+ bool caughtExpectedException = false;
+ try
+ {
+ using (Duplicati.Library.Utility.TempFile tempFile = new Duplicati.Library.Utility.TempFile())
+ {
+ backend.Get(string.Format("NonExistentFile-{0}", Guid.NewGuid()), tempFile.Name);
+ }
+ }
+ catch (FileMissingException ex)
+ {
+ Console.WriteLine("Caught expected FileMissingException");
+ caughtExpectedException = true;
+ }
+ catch (Exception ex)
+ {
+ Console.WriteLine("*** Retrieval of non-existent file failed: {0}", ex);
+ }
+
+ if (!caughtExpectedException)
+ {
+ Console.WriteLine("*** Retrieval of non-existent file should have failed with FileMissingException");
+ }
}
// Test quota retrieval
diff --git a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs index daeb7000e..a60a29028 100644 --- a/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/JSONWebHelper.cs @@ -304,20 +304,48 @@ namespace Duplicati.Library {
}
- public HttpWebResponse GetResponseWithoutException(string url, string method = null)
+ public HttpWebResponse GetResponseWithoutException(string url, object requestdata = null, string method = null)
{
- return GetResponseWithoutException(CreateRequest(url, method));
+ if (requestdata is string)
+ throw new ArgumentException("Cannot send string object as data");
+
+ if (method == null && requestdata != null)
+ method = "POST";
+
+ return GetResponseWithoutException(CreateRequest(url, method), requestdata);
}
- public HttpWebResponse GetResponseWithoutException(HttpWebRequest req)
+ public HttpWebResponse GetResponseWithoutException(HttpWebRequest req, object requestdata = null)
{
- return GetResponseWithoutException(new AsyncHttpRequest(req));
+ return GetResponseWithoutException(new AsyncHttpRequest(req), requestdata);
}
- public HttpWebResponse GetResponseWithoutException(AsyncHttpRequest req)
+ public HttpWebResponse GetResponseWithoutException(AsyncHttpRequest req, object requestdata = null)
{
try
{
+ if (requestdata != null)
+ {
+ if (requestdata is Stream stream)
+ {
+ req.Request.ContentLength = stream.Length;
+ if (string.IsNullOrEmpty(req.Request.ContentType))
+ req.Request.ContentType = "application/octet-stream";
+
+ using (var rs = req.GetRequestStream())
+ Library.Utility.Utility.CopyStream(stream, rs);
+ }
+ else
+ {
+ var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata));
+ req.Request.ContentLength = data.Length;
+ req.Request.ContentType = "application/json; charset=UTF-8";
+
+ using (var rs = req.GetRequestStream())
+ rs.Write(data, 0, data.Length);
+ }
+ }
+
return (HttpWebResponse)req.GetResponse();
}
catch(WebException wex)
@@ -329,6 +357,59 @@ namespace Duplicati.Library }
}
+ public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(string url, CancellationToken cancelToken, object requestdata = null, string method = null)
+ {
+ if (requestdata is string)
+ throw new ArgumentException("Cannot send string object as data");
+
+ if (method == null && requestdata != null)
+ method = "POST";
+
+ return await GetResponseWithoutExceptionAsync(CreateRequest(url, method), cancelToken, requestdata).ConfigureAwait(false);
+ }
+
+ public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(HttpWebRequest req, CancellationToken cancelToken, object requestdata = null)
+ {
+ return await GetResponseWithoutExceptionAsync(new AsyncHttpRequest(req), cancelToken, requestdata).ConfigureAwait(false);
+ }
+
+ public async Task<HttpWebResponse> GetResponseWithoutExceptionAsync(AsyncHttpRequest req, CancellationToken cancelToken, object requestdata = null)
+ {
+ try
+ {
+ if (requestdata != null)
+ {
+ if (requestdata is System.IO.Stream stream)
+ {
+ req.Request.ContentLength = stream.Length;
+ if (string.IsNullOrEmpty(req.Request.ContentType))
+ req.Request.ContentType = "application/octet-stream";
+
+ using (var rs = req.GetRequestStream())
+ await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken).ConfigureAwait(false);
+ }
+ else
+ {
+ var data = Encoding.UTF8.GetBytes(JsonConvert.SerializeObject(requestdata));
+ req.Request.ContentLength = data.Length;
+ req.Request.ContentType = "application/json; charset=UTF-8";
+
+ using (var rs = req.GetRequestStream())
+ await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
+ }
+ }
+
+ return (HttpWebResponse)req.GetResponse();
+ }
+ catch (WebException wex)
+ {
+ if (wex.Response is HttpWebResponse response)
+ return response;
+
+ throw;
+ }
+ }
+
public HttpWebResponse GetResponse(string url, object requestdata = null, string method = null)
{
if (requestdata is string)
@@ -405,7 +486,7 @@ namespace Duplicati.Library req.Request.ContentType = "application/octet-stream";
using (var rs = req.GetRequestStream())
- await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken);
+ await Utility.Utility.CopyStreamAsync(stream, rs, cancelToken).ConfigureAwait(false);
}
else
{
@@ -414,7 +495,7 @@ namespace Duplicati.Library req.Request.ContentType = "application/json; charset=UTF-8";
using (var rs = req.GetRequestStream())
- await rs.WriteAsync(data, 0, data.Length, cancelToken);
+ await rs.WriteAsync(data, 0, data.Length, cancelToken).ConfigureAwait(false);
}
}
diff --git a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs index 1e971cc58..007376cc0 100644 --- a/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs +++ b/Duplicati/Library/Backend/OAuthHelper/OAuthHelper.cs @@ -105,8 +105,13 @@ namespace Duplicati.Library public override HttpWebRequest CreateRequest(string url, string method = null)
{
+ return this.CreateRequest(url, method, false);
+ }
+
+ public HttpWebRequest CreateRequest(string url, string method, bool noAuthorization)
+ {
var r = base.CreateRequest(url, method);
- if (AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url))
+ if (!noAuthorization && AutoAuthHeader && !string.Equals(OAuthContextSettings.ServerURL, url))
r.Headers["Authorization"] = string.Format("Bearer {0}", AccessToken);
return r;
}
diff --git a/Duplicati/Library/Backend/OneDrive/Exceptions.cs b/Duplicati/Library/Backend/OneDrive/Exceptions.cs index d31562da2..0eb50882d 100644 --- a/Duplicati/Library/Backend/OneDrive/Exceptions.cs +++ b/Duplicati/Library/Backend/OneDrive/Exceptions.cs @@ -1,102 +1,222 @@ -using System; -using System.Net.Http; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -using Duplicati.Library.Utility; - -using Newtonsoft.Json; - -namespace Duplicati.Library.Backend.MicrosoftGraph -{ - public class MicrosoftGraphException : Exception - { - private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase); - - public MicrosoftGraphException(HttpResponseMessage response) - : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response) - { - } - - public MicrosoftGraphException(string message, HttpResponseMessage response) - : this(message, response, null) - { - } - - public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException) - : base(BuildFullMessage(message, response), innerException) - { - this.Response = response; - } - - public string RequestUrl => this.Response.RequestMessage.RequestUri.ToString(); - public HttpResponseMessage Response { get; private set; } - - protected static string ResponseToString(HttpResponseMessage response) - { - if (response != null) - { - // Start to read the content - Task<string> content = response.Content.ReadAsStringAsync(); - - // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included. - // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe. - // So we replace anything in the request that looks like the auth header with a safe version. - string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ"); - return string.Format("{0}\n{1}\n{2}", requestMessage, response, JsonConvert.SerializeObject(JsonConvert.DeserializeObject(content.Await()), Formatting.Indented)); - } - else - { - return null; - } - } - - private static string BuildFullMessage(string message, HttpResponseMessage response) - { - if (response != null) - { - return string.Format("{0}\n{1}", message, ResponseToString(response)); - } - else - { - return message; - } - } - } - - public class DriveItemNotFoundException : MicrosoftGraphException - { - public DriveItemNotFoundException(HttpResponseMessage response) - : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response) - { - } - } - - public class UploadSessionException : MicrosoftGraphException - { - public UploadSessionException( - HttpResponseMessage originalResponse, - int fragment, - int fragmentCount, - MicrosoftGraphException fragmentException) - : base( - string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), - originalResponse, - fragmentException) - { - this.Fragment = fragment; - this.FragmentCount = fragmentCount; - this.InnerException = fragmentException; - } - - public string CreateSessionRequestUrl => this.RequestUrl; - public HttpResponseMessage CreateSessionResponse => this.Response; - - public int Fragment { get; private set; } - public int FragmentCount { get; private set; } - public string FragmentRequestUrl => this.InnerException.RequestUrl; - public HttpResponseMessage FragmentResponse => this.InnerException.Response; - - public new MicrosoftGraphException InnerException { get; private set; } - } -} +using System;
+using System.IO;
+using System.Net;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+using System.Threading.Tasks;
+
+using Duplicati.Library.Utility;
+
+using Newtonsoft.Json;
+
+namespace Duplicati.Library.Backend.MicrosoftGraph
+{
+ public class MicrosoftGraphException : Exception
+ {
+ private static readonly Regex authorizationHeaderRemover = new Regex(@"Authorization:\s*Bearer\s+\S+", RegexOptions.IgnoreCase);
+
+ private readonly HttpResponseMessage responseMessage;
+ private readonly HttpWebResponse webResponse;
+
+ public MicrosoftGraphException(HttpResponseMessage response)
+ : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.ReasonPhrase, response.RequestMessage.RequestUri), response)
+ {
+ }
+
+ public MicrosoftGraphException(string message, HttpResponseMessage response)
+ : this(message, response, null)
+ {
+ }
+
+ public MicrosoftGraphException(string message, HttpResponseMessage response, Exception innerException)
+ : base(BuildFullMessage(message, response), innerException)
+ {
+ this.responseMessage = response;
+ }
+
+ public MicrosoftGraphException(HttpWebResponse response)
+ : this(string.Format("{0}: {1} error from request {2}", response.StatusCode, response.StatusDescription, response.ResponseUri), response)
+ {
+ }
+
+ public MicrosoftGraphException(string message, HttpWebResponse response)
+ : this(message, response, null)
+ {
+ }
+
+ public MicrosoftGraphException(string message, HttpWebResponse response, Exception innerException)
+ : base(BuildFullMessage(message, response), innerException)
+ {
+ this.webResponse = response;
+ }
+
+ public string RequestUrl
+ {
+ get
+ {
+ if (this.responseMessage != null)
+ {
+ return this.responseMessage.RequestMessage.RequestUri.ToString();
+ }
+ else
+ {
+ return this.webResponse.ResponseUri.ToString();
+ }
+ }
+ }
+
+ public HttpStatusCode StatusCode
+ {
+ get
+ {
+ if (this.responseMessage != null)
+ {
+ return this.responseMessage.StatusCode;
+ }
+ else
+ {
+ return this.webResponse.StatusCode;
+ }
+ }
+ }
+
+ public HttpResponseMessage Response { get; private set; }
+
+ protected static string ResponseToString(HttpResponseMessage response)
+ {
+ if (response != null)
+ {
+ // Start to read the content
+ using (Task<string> content = response.Content.ReadAsStringAsync())
+ {
+ // Since the exception message may be saved / sent in logs, we want to prevent the authorization header from being included.
+ // it wouldn't be as bad as recording the username/password in logs, since the token will expire, but it doesn't hurt to be safe.
+ // So we replace anything in the request that looks like the auth header with a safe version.
+ string requestMessage = authorizationHeaderRemover.Replace(response.RequestMessage.ToString(), "Authorization: Bearer ABC...XYZ");
+ return string.Format("{0}\n{1}\n{2}", requestMessage, response, PrettifyJson(content.Await()));
+ }
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ private static string BuildFullMessage(string message, HttpResponseMessage response)
+ {
+ if (response != null)
+ {
+ return string.Format("{0}\n{1}", message, ResponseToString(response));
+ }
+ else
+ {
+ return message;
+ }
+ }
+
+ protected static string ResponseToString(HttpWebResponse response)
+ {
+ if (response != null)
+ {
+ // Start to read the content
+ using (var responseStream = response.GetResponseStream())
+ using (var textReader = new StreamReader(responseStream))
+ {
+ return string.Format("{0}\n{1}", response, PrettifyJson(textReader.ReadToEnd()));
+ }
+ }
+ else
+ {
+ return null;
+ }
+ }
+
+ private static string BuildFullMessage(string message, HttpWebResponse response)
+ {
+ if (response != null)
+ {
+ return string.Format("{0}\n{1}", message, ResponseToString(response));
+ }
+ else
+ {
+ return message;
+ }
+ }
+
+ private static string PrettifyJson(string json)
+ {
+ if (string.IsNullOrWhiteSpace(json))
+ {
+ return json;
+ }
+
+ if (json[0] == '<')
+ {
+ // It looks like some errors return xml bodies instead of JSON.
+ // If this looks like it might be one of those, don't even bother parsing the JSON.
+ return json;
+ }
+
+ try
+ {
+ return JsonConvert.SerializeObject(JsonConvert.DeserializeObject(json), Formatting.Indented);
+ }
+ catch (Exception)
+ {
+ // Maybe this wasn't JSON..
+ return json;
+ }
+ }
+ }
+
+ public class DriveItemNotFoundException : MicrosoftGraphException
+ {
+ public DriveItemNotFoundException(HttpResponseMessage response)
+ : base(string.Format("Item at {0} was not found", response?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"), response)
+ {
+ }
+
+ public DriveItemNotFoundException(HttpWebResponse response)
+ : base(string.Format("Item at {0} was not found", response?.ResponseUri?.ToString() ?? "<unknown>"), response)
+ {
+ }
+ }
+
+ public class UploadSessionException : MicrosoftGraphException
+ {
+ public UploadSessionException(
+ HttpResponseMessage originalResponse,
+ int fragment,
+ int fragmentCount,
+ MicrosoftGraphException fragmentException)
+ : base(
+ string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.RequestMessage?.RequestUri?.ToString() ?? "<unknown>"),
+ originalResponse,
+ fragmentException)
+ {
+ this.Fragment = fragment;
+ this.FragmentCount = fragmentCount;
+ this.InnerException = fragmentException;
+ }
+
+ public UploadSessionException(
+ HttpWebResponse originalResponse,
+ int fragment,
+ int fragmentCount,
+ MicrosoftGraphException fragmentException)
+ : base(
+ string.Format("Error uploading fragment {0} of {1} for {2}", fragment, fragmentCount, originalResponse?.ResponseUri?.ToString() ?? "<unknown>"),
+ originalResponse,
+ fragmentException)
+ {
+ this.Fragment = fragment;
+ this.FragmentCount = fragmentCount;
+ this.InnerException = fragmentException;
+ }
+
+ public int Fragment { get; private set; }
+ public int FragmentCount { get; private set; }
+
+ public new MicrosoftGraphException InnerException { get; private set; }
+ }
+}
diff --git a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs index 1983d43e1..c8e533fc7 100644 --- a/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs +++ b/Duplicati/Library/Backend/OneDrive/MicrosoftGraphBackend.cs @@ -39,6 +39,7 @@ namespace Duplicati.Library.Backend private const string UPLOAD_SESSION_FRAGMENT_SIZE_OPTION = "fragment-size";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION = "fragment-retry-count";
private const string UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION = "fragment-retry-delay";
+ private const string USE_HTTP_CLIENT = "use-http-client";
private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT = 5;
private const int UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY = 1000;
@@ -68,12 +69,27 @@ namespace Duplicati.Library.Backend /// </summary>
private const int UPLOAD_SESSION_FRAGMENT_MULTIPLE_SIZE = 320 * 1024;
+ /// <summary>
+ /// Whether to use the HttpClient class for HTTP requests.
+ /// Default is false when running under Mono (as it seems it might be causing a memory leak in some environments / versions)
+ /// but true in other cases (where these memory leaks haven't been reproduced).
+ /// </summary>
+ private static readonly bool USE_HTTP_CLIENT_DEFAULT = Utility.Utility.IsMono ? false : true;
+
private static readonly HttpMethod PatchMethod = new HttpMethod("PATCH");
+ /// <summary>
+ /// Dummy UploadSession given as an empty body to createUploadSession requests when using the OAuthHelper instead of the OAuthHttpClient.
+ /// The API expects a ContentLength to be specified, but the body content is optional.
+ /// Passing an empty object (or specifying the ContentLength explicitly) bypasses this error.
+ /// </summary>
+ private static readonly UploadSession dummyUploadSession = new UploadSession();
+
protected delegate string DescriptionTemplateDelegate(string mssadescription, string mssalink, string msopdescription, string msoplink);
private readonly JsonSerializer m_serializer = new JsonSerializer();
private readonly OAuthHttpClient m_client;
+ private readonly OAuthHelper m_oAuthHelper;
private readonly int fragmentSize;
private readonly int fragmentRetryCount;
private readonly int fragmentRetryDelay; // In milliseconds
@@ -119,8 +135,27 @@ namespace Duplicati.Library.Backend this.fragmentRetryDelay = UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY;
}
- this.m_client = new OAuthHttpClient(authid, protocolKey);
- this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);
+ bool useHttpClient;
+ string useHttpClientStr;
+ if (options.TryGetValue(USE_HTTP_CLIENT, out useHttpClientStr))
+ {
+ useHttpClient = Utility.Utility.ParseBool(useHttpClientStr, USE_HTTP_CLIENT_DEFAULT);
+ }
+ else
+ {
+ useHttpClient = USE_HTTP_CLIENT_DEFAULT;
+ }
+
+ if (useHttpClient)
+ {
+ this.m_client = new OAuthHttpClient(authid, protocolKey);
+ this.m_client.BaseAddress = new System.Uri(BASE_ADDRESS);
+ }
+ else
+ {
+ this.m_oAuthHelper = new OAuthHelper(authid, protocolKey);
+ this.m_oAuthHelper.AutoAuthHeader = true;
+ }
// Extract out the path to the backup root folder from the given URI. Since this can be an expensive operation,
// we will cache the value using a lazy initializer.
@@ -153,6 +188,7 @@ namespace Duplicati.Library.Backend new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_SIZE_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentSizeShort, Strings.MicrosoftGraph.FragmentSizeLong, Library.Utility.Utility.FormatSizeString(UPLOAD_SESSION_FRAGMENT_DEFAULT_SIZE)),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_COUNT_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryCountShort, Strings.MicrosoftGraph.FragmentRetryCountLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_COUNT.ToString()),
new CommandLineArgument(UPLOAD_SESSION_FRAGMENT_RETRY_DELAY_OPTION, CommandLineArgument.ArgumentType.Integer, Strings.MicrosoftGraph.FragmentRetryDelayShort, Strings.MicrosoftGraph.FragmentRetryDelayLong, UPLOAD_SESSION_FRAGMENT_DEFAULT_RETRY_DELAY.ToString()),
+ new CommandLineArgument(USE_HTTP_CLIENT, CommandLineArgument.ArgumentType.Boolean, Strings.MicrosoftGraph.UseHttpClientShort, Strings.MicrosoftGraph.UseHttpClientLong, USE_HTTP_CLIENT_DEFAULT.ToString()),
}
.Concat(this.AdditionalSupportedCommands).ToList();
}
@@ -170,13 +206,23 @@ namespace Duplicati.Library.Backend // To get the upload session endpoint, we can start an upload session and then immediately cancel it.
// We pick a random file name (using a guid) to make sure we don't conflict with an existing file
string dnsTestFile = string.Format("DNSNameTest-{0}", Guid.NewGuid());
- UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), null);
+ UploadSession uploadSession = this.Post<UploadSession>(string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(dnsTestFile)), MicrosoftGraphBackend.dummyUploadSession);
// Canceling an upload session is done by sending a DELETE to the upload URL
- using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl))
- using (var response = this.m_client.SendAsync(request).Await())
+ if (this.m_client != null)
{
- this.CheckResponse(response);
+ using (var request = new HttpRequestMessage(HttpMethod.Delete, uploadSession.UploadUrl))
+ using (var response = this.m_client.SendAsync(request).Await())
+ {
+ this.CheckResponse(response);
+ }
+ }
+ else
+ {
+ using (var response = this.m_oAuthHelper.GetResponseWithoutException(uploadSession.UploadUrl, MicrosoftGraphBackend.dummyUploadSession, HttpMethod.Delete.ToString()))
+ {
+ this.CheckResponse(response);
+ }
}
this.dnsNames = new[]
@@ -238,7 +284,18 @@ namespace Duplicati.Library.Backend private string DrivePrefix
{
- get { return this.ApiVersion + this.DrivePath; }
+ get
+ {
+ if (this.m_client != null)
+ {
+ return this.ApiVersion + this.DrivePath;
+ }
+ else
+ {
+ // When not using the HttpClient for requests, the base address needs to be included in this prefix
+ return BASE_ADDRESS + this.ApiVersion + this.DrivePath;
+ }
+ }
}
public void CreateFolder()
@@ -297,12 +354,27 @@ namespace Duplicati.Library.Backend {
try
{
- using (var response = this.m_client.GetAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await())
+ string getUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
+ if (this.m_client != null)
{
- this.CheckResponse(response);
- using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())
+ using (var response = this.m_client.GetAsync(getUrl).Await())
{
- responseStream.CopyTo(stream);
+ this.CheckResponse(response);
+ using (Stream responseStream = response.Content.ReadAsStreamAsync().Await())
+ {
+ responseStream.CopyTo(stream);
+ }
+ }
+ }
+ else
+ {
+ using (var response = this.m_oAuthHelper.GetResponseWithoutException(getUrl))
+ {
+ this.CheckResponse(response);
+ using (Stream responseStream = response.GetResponseStream())
+ {
+ responseStream.CopyTo(stream);
+ }
}
}
}
@@ -339,10 +411,22 @@ namespace Duplicati.Library.Backend // PUT only supports up to 4 MB file uploads. There's a separate process for larger files.
if (stream.Length < PUT_MAX_SIZE)
{
- using (StreamContent streamContent = new StreamContent(stream))
+ string putUrl = string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
+ if (this.m_client != null)
+ {
+ using (StreamContent streamContent = new StreamContent(stream))
+ {
+ streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
+ using (var response = await this.m_client.PutAsync(putUrl, streamContent, cancelToken).ConfigureAwait(false))
+ {
+ // Make sure this response is a valid drive item, though we don't actually use it for anything currently.
+ this.ParseResponse<DriveItem>(response);
+ }
+ }
+ }
+ else
{
- streamContent.Headers.ContentType = new MediaTypeHeaderValue("application/octet-stream");
- using (var response = await m_client.PutAsync(string.Format("{0}/root:{1}{2}:/content", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename)), streamContent, cancelToken).ConfigureAwait(false))
+ using (var response = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(putUrl, cancelToken, stream, HttpMethod.Put.ToString()).ConfigureAwait(false))
{
// Make sure this response is a valid drive item, though we don't actually use it for anything currently.
this.ParseResponse<DriveItem>(response);
@@ -356,9 +440,11 @@ namespace Duplicati.Library.Backend // The documentation seems somewhat contradictory - it states that uploads must be done sequentially,
// but also states that the nextExpectedRanges value returned may indicate multiple ranges...
// For now, this plays it safe and does a sequential upload.
- using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))))
+ string createSessionUrl = string.Format("{0}/root:{1}{2}:/createUploadSession", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
+ if (this.m_client != null)
{
- using (HttpResponseMessage createSessionResponse = await m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false))
+ using (HttpRequestMessage createSessionRequest = new HttpRequestMessage(HttpMethod.Post, createSessionUrl))
+ using (HttpResponseMessage createSessionResponse = await this.m_client.SendAsync(createSessionRequest, cancelToken).ConfigureAwait(false))
{
UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);
@@ -385,7 +471,7 @@ namespace Duplicati.Library.Backend try
{
// The uploaded put requests will error if they are authenticated
- using (HttpResponseMessage response = await m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))
+ using (HttpResponseMessage response = await this.m_client.SendAsync(request, false, cancelToken).ConfigureAwait(false))
{
// Note: On the last request, the json result includes the default properties of the item that was uploaded
this.ParseResponse<UploadSession>(response);
@@ -400,20 +486,20 @@ namespace Duplicati.Library.Backend // We've used up all our retry attempts
throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
}
- else if ((int)ex.Response.StatusCode >= 500 && (int)ex.Response.StatusCode < 600)
+ else if ((int)ex.StatusCode >= 500 && (int)ex.StatusCode < 600)
{
// If a 5xx error code is hit, we should use an exponential backoff strategy before retrying.
// To make things simpler, we just use the current attempt number as the exponential factor.
Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await()
continue;
}
- else if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+ else if (ex.StatusCode == HttpStatusCode.NotFound)
{
// 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.
// Instead we'll let the caller re-attempt the whole file.
throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
}
- else if ((int)ex.Response.StatusCode >= 400 && (int)ex.Response.StatusCode < 500)
+ else if ((int)ex.StatusCode >= 400 && (int)ex.StatusCode < 500)
{
// If a 4xx error code is hit, we should retry without the backoff attempt
continue;
@@ -432,6 +518,83 @@ namespace Duplicati.Library.Backend }
}
}
+ else
+ {
+ using (HttpWebResponse createSessionResponse = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(createSessionUrl, cancelToken, MicrosoftGraphBackend.dummyUploadSession, HttpMethod.Post.ToString()))
+ {
+ UploadSession uploadSession = this.ParseResponse<UploadSession>(createSessionResponse);
+
+ // If the stream's total length is less than the chosen fragment size, then we should make the buffer only as large as the stream.
+ int bufferSize = (int)Math.Min(this.fragmentSize, stream.Length);
+
+ byte[] fragmentBuffer = new byte[bufferSize];
+ int read = 0;
+ for (int offset = 0; offset < stream.Length; offset += read)
+ {
+ read = await stream.ReadAsync(fragmentBuffer, 0, bufferSize, cancelToken).ConfigureAwait(false);
+
+ int retryCount = this.fragmentRetryCount;
+ for (int attempt = 0; attempt < retryCount; attempt++)
+ {
+ // The uploaded put requests will error if they are authenticated
+ var request = new AsyncHttpRequest(this.m_oAuthHelper.CreateRequest(uploadSession.UploadUrl, HttpMethod.Put.ToString(), true));
+ request.Request.ContentLength = read;
+ request.Request.Headers.Set(HttpRequestHeader.ContentRange, new ContentRangeHeaderValue(offset, offset + read - 1, stream.Length).ToString());
+ request.Request.ContentType = "application/octet-stream";
+
+ using (var requestStream = request.GetRequestStream(read))
+ {
+ await requestStream.WriteAsync(fragmentBuffer, 0, read, cancelToken);
+ }
+
+ try
+ {
+ using (var response = await this.m_oAuthHelper.GetResponseWithoutExceptionAsync(request, cancelToken))
+ {
+ // Note: On the last request, the json result includes the default properties of the item that was uploaded
+ this.ParseResponse<UploadSession>(response);
+ }
+ }
+ catch (MicrosoftGraphException ex)
+ {
+ // Error handling based on recommendations here:
+ // https://docs.microsoft.com/en-us/onedrive/developer/rest-api/api/driveitem_createuploadsession#best-practices
+ if (attempt >= retryCount - 1)
+ {
+ // We've used up all our retry attempts
+ throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
+ }
+ else if ((int)ex.StatusCode >= 500 && (int)ex.StatusCode < 600)
+ {
+ // If a 5xx error code is hit, we should use an exponential backoff strategy before retrying.
+ // To make things simpler, we just use the current attempt number as the exponential factor.
+ Thread.Sleep((int)Math.Pow(2, attempt) * this.fragmentRetryDelay); // If this is changed to use tasks, this should be changed to Task.Await()
+ continue;
+ }
+ else if (ex.StatusCode == HttpStatusCode.NotFound)
+ {
+ // 404 is a special case indicating the upload session no longer exists, so the fragment shouldn't be retried.
+ // Instead we'll let the caller re-attempt the whole file.
+ throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
+ }
+ else if ((int)ex.StatusCode >= 400 && (int)ex.StatusCode < 500)
+ {
+ // If a 4xx error code is hit, we should retry without the backoff attempt
+ continue;
+ }
+ else
+ {
+ // Other errors should be rethrown
+ throw new UploadSessionException(createSessionResponse, offset / bufferSize, (int)Math.Ceiling((double)stream.Length / bufferSize), ex);
+ }
+ }
+
+ // If we successfully sent this piece, then we can break out of the retry loop
+ break;
+ }
+ }
+ }
+ }
}
}
@@ -439,9 +602,20 @@ namespace Duplicati.Library.Backend {
try
{
- using (var response = this.m_client.DeleteAsync(string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename))).Await())
+ string deleteUrl = string.Format("{0}/root:{1}{2}", this.DrivePrefix, this.RootPath, NormalizeSlashes(remotename));
+ if (this.m_client != null)
+ {
+ using (var response = this.m_client.DeleteAsync(deleteUrl).Await())
+ {
+ this.CheckResponse(response);
+ }
+ }
+ else
{
- this.CheckResponse(response);
+ using (var response = this.m_oAuthHelper.GetResponseWithoutException(deleteUrl, null, HttpMethod.Delete.ToString()))
+ {
+ this.CheckResponse(response);
+ }
}
}
catch (DriveItemNotFoundException ex)
@@ -498,18 +672,38 @@ namespace Duplicati.Library.Backend private T SendRequest<T>(HttpMethod method, string url)
{
- using (var request = new HttpRequestMessage(method, url))
+ if (this.m_client != null)
{
- return this.SendRequest<T>(request);
+ using (var request = new HttpRequestMessage(method, url))
+ {
+ return this.SendRequest<T>(request);
+ }
+ }
+ else
+ {
+ using (var response = this.m_oAuthHelper.GetResponseWithoutException(url, null, method.ToString()))
+ {
+ return this.ParseResponse<T>(response);
+ }
}
}
private T SendRequest<T>(HttpMethod method, string url, T body) where T : class
{
- using (var request = new HttpRequestMessage(method, url))
- using (request.Content = this.PrepareContent(body))
+ if (this.m_client != null)
{
- return this.SendRequest<T>(request);
+ using (var request = new HttpRequestMessage(method, url))
+ using (request.Content = this.PrepareContent(body))
+ {
+ return this.SendRequest<T>(request);
+ }
+ }
+ else
+ {
+ using (var response = this.m_oAuthHelper.GetResponseWithoutException(url, body, method.ToString()))
+ {
+ return this.ParseResponse<T>(response);
+ }
}
}
@@ -520,7 +714,7 @@ namespace Duplicati.Library.Backend return this.ParseResponse<T>(response);
}
}
-
+
private IEnumerable<T> Enumerate<T>(string url)
{
string nextUrl = url;
@@ -563,6 +757,23 @@ namespace Duplicati.Library.Backend }
}
+ private void CheckResponse(HttpWebResponse response)
+ {
+ if (!((int)response.StatusCode >= 200 && (int)response.StatusCode < 300))
+ {
+ if (response.StatusCode == HttpStatusCode.NotFound)
+ {
+ // It looks like this is an 'item not found' exception, so wrap it in a new exception class to make it easier to pick things out.
+ throw new DriveItemNotFoundException(response);
+ }
+ else
+ {
+ // Throw a wrapper exception to make it easier for the caller to look at specific status codes, etc.
+ throw new MicrosoftGraphException(response);
+ }
+ }
+ }
+
private T ParseResponse<T>(HttpResponseMessage response)
{
this.CheckResponse(response);
@@ -574,6 +785,17 @@ namespace Duplicati.Library.Backend }
}
+ private T ParseResponse<T>(HttpWebResponse response)
+ {
+ this.CheckResponse(response);
+ using (Stream responseStream = response.GetResponseStream())
+ using (StreamReader reader = new StreamReader(responseStream))
+ using (JsonTextReader jsonReader = new JsonTextReader(reader))
+ {
+ return this.m_serializer.Deserialize<T>(jsonReader);
+ }
+ }
+
/// <summary>
/// Normalizes the slashes in a url fragment. For example:
/// "" => ""
diff --git a/Duplicati/Library/Backend/OneDrive/Strings.cs b/Duplicati/Library/Backend/OneDrive/Strings.cs index a1b0a05c5..d2892d0c9 100644 --- a/Duplicati/Library/Backend/OneDrive/Strings.cs +++ b/Duplicati/Library/Backend/OneDrive/Strings.cs @@ -12,6 +12,8 @@ namespace Duplicati.Library.Backend.Strings public static string FragmentRetryCountLong { get { return LC.L(@"Number of retry attempts made for each fragment before failing the overall file upload"); } }
public static string FragmentRetryDelayShort { get { return LC.L(@"Millisecond delay between fragment errors"); } }
public static string FragmentRetryDelayLong { get { return LC.L(@"Amount of time (in milliseconds) to wait between failures when uploading fragments"); } }
+ public static string UseHttpClientShort { get { return LC.L(@"Whether the HttpClient class should be used"); } }
+ public static string UseHttpClientLong { get { return LC.L(@"Whether the HttpClient class should be used to perform HTTP requests"); } }
}
internal static class OneDriveV2
|