diff options
| author | Tyler Gill <tyler.gill@byu.net> | 2017-09-26 08:17:45 +0300 |
|---|---|---|
| committer | Tyler Gill <tyler.gill@byu.net> | 2017-09-26 08:17:45 +0300 |
| commit | 38883285eee5598e37d56e6ddc417fad7d872ed3 (patch) | |
| tree | d542798f8594aca02d954128f0fe63f32f28b512 /Duplicati/Library/Backend | |
| parent | 6c921ab3ec878a99262150b80b7f29713700c11e (diff) | |
Change IBackend.List() to return IEnumerable instead of List
By changing to IEnumerable, it is possible to iterate only a portion of the list, which is useful when not all entries are needed (e.g., when testing a connection).
All existing backends have been updated, and any which were able to be changed to yield return results in a straightforward way now do.
Many backends had a try/catch in the List() method. Due to the fact that yield returns can't be placed within a try/catch block, these have been refactored to either scope the try/catch to the parts that (should) be the only places throwing exceptions, so that exceptions are still caught and handled.
Note that lazy evaluation may cause some changes in behavior - exceptions that were previously thrown at the point of invokation of List() may now be thrown while it is being enumerated.
I believe this will not be problematic though, as the only well-known exception seems to be FolderMissingException, which should be thrown by Test(), but TestList() attempts to enumerate the list to force this exception.
Any places that require the legacy behavior can get it by simply converting the lazy enumerable to a List()
Diffstat (limited to 'Duplicati/Library/Backend')
23 files changed, 368 insertions, 369 deletions
diff --git a/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs b/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs index 9cdcc09b9..032854043 100644 --- a/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs +++ b/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs @@ -194,17 +194,17 @@ namespace Duplicati.Library.Backend.AlternativeFTP }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
return List("");
}
- public List<IFileEntry> List(string filename)
+ public IEnumerable<IFileEntry> List(string filename)
{
return List(filename, false);
}
- private List<IFileEntry> List(string filename, bool stripFile)
+ private IEnumerable<IFileEntry> List(string filename, bool stripFile)
{
var list = new List<IFileEntry>();
string remotePath = filename;
diff --git a/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs b/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs index e24a949fa..b68319333 100644 --- a/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs +++ b/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs @@ -336,7 +336,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive #endregion
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
EnforceConsistencyDelay(RemoteOperation.List);
@@ -416,7 +416,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive }
public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
{
diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 2e200c8fd..d55eb17b0 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Backend.AzureBlob get { return true; } } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { return _azureBlob.ListContainerEntries(); } @@ -161,7 +161,7 @@ namespace Duplicati.Library.Backend.AzureBlob public void Test() { - List(); + this.TestList(); } public void CreateFolder() diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 383e2e009..682e05e23 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -285,7 +285,7 @@ namespace Duplicati.Library.Backend.Backblaze }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
m_filecache = null;
var cache = new Dictionary<string, List<FileEntity>>();
@@ -380,7 +380,7 @@ namespace Duplicati.Library.Backend.Backblaze public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 51e3c838e..124a51fe7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -252,12 +252,11 @@ namespace Duplicati.Library.Backend.Box #region IBackend implementation
- public System.Collections.Generic.List<IFileEntry> List()
+ public System.Collections.Generic.IEnumerable<IFileEntry> List()
{
- return (
+ return
from n in PagedFileListResponse(CurrentFolder, false)
- select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" }
- ).ToList();
+ select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" };
}
public void Put(string remotename, string filename)
@@ -295,7 +294,7 @@ namespace Duplicati.Library.Backend.Box public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index 1b33756d3..29cbc2ba1 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -19,6 +19,7 @@ #endregion
using System;
using System.Collections.Generic;
+using System.Linq;
using System.Text;
using System.Net;
using Duplicati.Library.Interface;
@@ -128,9 +129,8 @@ namespace Duplicati.Library.Backend get { return "cloudfiles"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var files = new List<IFileEntry>();
string extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
string markerUrl = "";
@@ -182,7 +182,7 @@ namespace Duplicati.Library.Backend mod = new DateTime();
lastItemName = name;
- files.Add(new FileEntry(name, size, mod, mod));
+ yield return new FileEntry(name, size, mod, mod);
}
repeat = lst.Count == ITEM_LIST_LIMIT;
@@ -191,8 +191,6 @@ namespace Duplicati.Library.Backend markerUrl = "&marker=" + Library.Utility.Uri.UrlEncode(lastItemName);
} while (repeat);
-
- return files;
}
public void Put(string remotename, string filename)
@@ -263,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test()
{
//The "Folder not found" is not detectable :(
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/Dropbox/Dropbox.cs b/Duplicati/Library/Backend/Dropbox/Dropbox.cs index 8a70684a0..76e555c0a 100644 --- a/Duplicati/Library/Backend/Dropbox/Dropbox.cs +++ b/Duplicati/Library/Backend/Dropbox/Dropbox.cs @@ -67,35 +67,37 @@ namespace Duplicati.Library.Backend return ife;
}
-
- public List<IFileEntry> List()
+
+ public IEnumerable<IFileEntry> List()
{
try
{
- var list = new List<IFileEntry>();
- var lfr = dbx.ListFiles(m_path);
-
- foreach (var md in lfr.entries)
- list.Add(ParseEntry(md));
-
- while (lfr.has_more)
- {
- lfr = dbx.ListFilesContinue(lfr.cursor);
- foreach (var md in lfr.entries)
- list.Add(ParseEntry(md));
- }
-
- return list;
+ return ListWithoutExceptionCatch();
}
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found")
throw new FolderMissingException();
-
+
throw;
}
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ var lfr = dbx.ListFiles(m_path);
+
+ foreach (var md in lfr.entries)
+ yield return ParseEntry(md);
+
+ while (lfr.has_more)
+ {
+ lfr = dbx.ListFilesContinue(lfr.cursor);
+ foreach (var md in lfr.entries)
+ yield return ParseEntry(md);
+ }
+ }
+
public void Put(string remotename, string filename)
{
using(FileStream fs = System.IO.File.OpenRead(filename))
@@ -136,7 +138,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 1aea8537d..6ac888979 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -170,12 +170,12 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
return List("");
}
-
- public List<IFileEntry> List(string filename)
+
+ public IEnumerable<IFileEntry> List(string filename)
{
var req = CreateRequest(filename);
req.Method = System.Net.WebRequestMethods.Ftp.ListDirectoryDetails;
@@ -183,21 +183,7 @@ namespace Duplicati.Library.Backend try
{
- var lst = new List<IFileEntry>();
- var areq = new Utility.AsyncHttpRequest(req);
- using (var resp = areq.GetResponse())
- using (var rs = areq.GetResponseStream())
- using (var sr = new System.IO.StreamReader(new StreamReadHelper(rs)))
- {
- string line;
- while ((line = sr.ReadLine()) != null)
- {
- FileEntry f = ParseLine(line);
- if (f != null)
- lst.Add(f);
- }
- }
- return lst;
+ return ListWithoutExceptionCatch(req);
}
catch (System.Net.WebException wex)
{
@@ -208,6 +194,23 @@ namespace Duplicati.Library.Backend }
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch(System.Net.FtpWebRequest req)
+ {
+ var areq = new Utility.AsyncHttpRequest(req);
+ using (var resp = areq.GetResponse())
+ using (var rs = areq.GetResponseStream())
+ using (var sr = new System.IO.StreamReader(new StreamReadHelper(rs)))
+ {
+ string line;
+ while ((line = sr.ReadLine()) != null)
+ {
+ FileEntry f = ParseLine(line);
+ if (f != null)
+ yield return f;
+ }
+ }
+ }
+
public void Put(string remotename, System.IO.Stream input)
{
System.Net.FtpWebRequest req = null;
@@ -227,7 +230,7 @@ namespace Duplicati.Library.Backend if (m_listVerify)
{
- List<IFileEntry> files = List(remotename);
+ IEnumerable<IFileEntry> files = List(remotename);
foreach(IFileEntry fe in files)
if (fe.Name.Equals(remotename) || fe.Name.EndsWith("/" + remotename) || fe.Name.EndsWith("\\" + remotename))
{
@@ -308,7 +311,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index cafba1170..8a846594b 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -164,30 +164,26 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- List<IFileEntry> ls = new List<IFileEntry>();
-
PreAuthenticate();
if (!System.IO.Directory.Exists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
- foreach (string s in System.IO.Directory.GetFiles(m_path))
+ foreach (string s in System.IO.Directory.EnumerateFiles(m_path))
{
System.IO.FileInfo fi = new System.IO.FileInfo(s);
- ls.Add(new FileEntry(fi.Name, fi.Length, fi.LastAccessTime, fi.LastWriteTime));
+ yield return new FileEntry(fi.Name, fi.Length, fi.LastAccessTime, fi.LastWriteTime);
}
- foreach (string s in System.IO.Directory.GetDirectories(m_path))
+ foreach (string s in System.IO.Directory.EnumerateDirectories(m_path))
{
System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(s);
FileEntry fe = new FileEntry(di.Name, 0, di.LastAccessTime, di.LastWriteTime);
fe.IsFolder = true;
- ls.Add(fe);
+ yield return fe;
}
-
- return ls;
}
#if DEBUG_RETRY
@@ -265,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index dcdeb64db..663bd274b 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -98,8 +98,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage m_oauth = new OAuthHelper(authid, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
-
-
+
private class ListBucketResponse
{
@@ -130,39 +129,13 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public string location { get; set; }
public string storageClass { get; set; }
}
+
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var res = new List<IFileEntry>();
- string token = null;
- do
- {
- var url = string.Format("{0}/b/{1}/o?prefix={2}", API_URL, m_bucket, Library.Utility.Uri.UrlEncode(m_prefix));
- if (!string.IsNullOrEmpty(token))
- url += string.Format("&pageToken={0}", token);
- var resp = m_oauth.ReadJSONResponse<ListBucketResponse>(url);
-
- if (resp.items != null)
- foreach(var f in resp.items)
- {
- var name = f.name;
- if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
- name = name.Substring(m_prefix.Length);
- if (f.size == null)
- res.Add(new FileEntry(name));
- else if (f.updated == null)
- res.Add(new FileEntry(name, f.size.Value));
- else
- res.Add(new FileEntry(name, f.size.Value, f.updated.Value, f.updated.Value));
- }
-
- token = resp.nextPageToken;
-
- } while(!string.IsNullOrEmpty(token));
-
- return res;
+ return this.ListWithoutExceptionCatch();
}
catch (WebException wex)
{
@@ -173,6 +146,35 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage }
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ string token = null;
+ do
+ {
+ var url = string.Format("{0}/b/{1}/o?prefix={2}", API_URL, m_bucket, Library.Utility.Uri.UrlEncode(m_prefix));
+ if (!string.IsNullOrEmpty(token))
+ url += string.Format("&pageToken={0}", token);
+ var resp = m_oauth.ReadJSONResponse<ListBucketResponse>(url);
+
+ if (resp.items != null)
+ foreach (var f in resp.items)
+ {
+ var name = f.name;
+ if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
+ name = name.Substring(m_prefix.Length);
+ if (f.size == null)
+ yield return new FileEntry(name);
+ else if (f.updated == null)
+ yield return new FileEntry(name, f.size.Value);
+ else
+ yield return new FileEntry(name, f.size.Value, f.updated.Value, f.updated.Value);
+ }
+
+ token = resp.nextPageToken;
+
+ } while (!string.IsNullOrEmpty(token));
+ }
+
public void Put(string remotename, string filename)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
@@ -195,7 +197,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index c4e6af992..736044bbe 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -189,44 +189,14 @@ namespace Duplicati.Library.Backend.GoogleDrive #region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var res = new List<IFileEntry>();
m_filecache.Clear();
- foreach(var n in ListFolder(CurrentFolderId))
- {
- FileEntry fe = null;
-
- if (n.fileSize == null)
- fe = new FileEntry(n.title);
- else if (n.modifiedDate == null)
- fe = new FileEntry(n.title, n.fileSize.Value);
- else
- fe = new FileEntry(n.title, n.fileSize.Value, n.modifiedDate.Value, n.modifiedDate.Value);
-
- if (fe != null)
- {
- fe.IsFolder = FOLDER_MIMETYPE.Equals(n.mimeType, StringComparison.OrdinalIgnoreCase);
- res.Add(fe);
-
- if (!fe.IsFolder)
- {
- GoogleDriveFolderItem[] lst;
- if (!m_filecache.TryGetValue(fe.Name, out lst))
- m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
- else
- {
- Array.Resize(ref lst, lst.Length + 1);
- lst[lst.Length - 1] = n;
- }
- }
- }
- }
-
- return res;
+ // For now, this class assumes that List() fully populates the file cache
+ return ListWithoutExceptionCatch().ToList();
}
catch
{
@@ -234,7 +204,42 @@ namespace Duplicati.Library.Backend.GoogleDrive throw;
}
+ }
+
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ foreach (var n in ListFolder(CurrentFolderId))
+ {
+ FileEntry fe = null;
+
+ if (n.fileSize == null)
+ fe = new FileEntry(n.title);
+ else if (n.modifiedDate == null)
+ fe = new FileEntry(n.title, n.fileSize.Value);
+ else
+ fe = new FileEntry(n.title, n.fileSize.Value, n.modifiedDate.Value, n.modifiedDate.Value);
+
+ if (fe != null)
+ {
+ fe.IsFolder = FOLDER_MIMETYPE.Equals(n.mimeType, StringComparison.OrdinalIgnoreCase);
+
+ if (!fe.IsFolder)
+ {
+ GoogleDriveFolderItem[] lst;
+ if (!m_filecache.TryGetValue(fe.Name, out lst))
+ {
+ m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
+ }
+ else
+ {
+ Array.Resize(ref lst, lst.Length + 1);
+ lst[lst.Length - 1] = n;
+ }
+ }
+ yield return fe;
+ }
+ }
}
public void Put(string remotename, string filename)
@@ -274,7 +279,7 @@ namespace Duplicati.Library.Backend.GoogleDrive public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/HubiC/HubiCBackend.cs b/Duplicati/Library/Backend/HubiC/HubiCBackend.cs index 748842d59..1f8fb03c0 100644 --- a/Duplicati/Library/Backend/HubiC/HubiCBackend.cs +++ b/Duplicati/Library/Backend/HubiC/HubiCBackend.cs @@ -21,6 +21,6 @@ namespace Duplicati.Library.Backend.HubiC public class HubiCBackend : IBackend, IStreamingBackend
{
private const string AUTHID_OPTION = "authid";
private const string HUBIC_API_URL = "https://api.hubic.com/1.0/";
private const string HUBIC_API_CREDENTIAL_URL = HUBIC_API_URL + "account/credentials";
private OpenStackStorage m_openstack;
private class HubiCAuthResponse
{
public string token { get; set; }
public string endpoint { get; set; }
public DateTime? expires { get; set;}
}
private class OpenStackHelper : OpenStackStorage
{
private OAuthHelper m_helper;
private HubiCAuthResponse m_token;
public OpenStackHelper(string authid, string url)
: base(url, MockOptions())
{
m_helper = new OAuthHelper(authid, "hubic") { AutoAuthHeader = true };
}
private static Dictionary<string, string> MockOptions()
{
var res = new Dictionary<string, string>();
res["openstack-authuri"] = "invalid://dont-use";
res["openstack-apikey"] = "invalid";
res["auth-username"] = "invalid";
return res;
}
private HubiCAuthResponse AuthToken
{
get
{
if (m_token == null || (m_token.expires != null && (m_token.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
m_token = m_helper.ReadJSONResponse<HubiCAuthResponse>(HUBIC_API_CREDENTIAL_URL);
return m_token;
}
}
protected override string AccessToken
{
get { return AuthToken.token; }
}
protected override string SimpleStorageEndPoint
{
get { return AuthToken.endpoint; }
}
}
public HubiCBackend()
{
}
-
public HubiCBackend(string url, Dictionary<string, string> options)
{
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
m_openstack = new OpenStackHelper(authid, url);
}
#region IStreamingBackend implementation
public void Put(string remotename, System.IO.Stream stream)
{
m_openstack.Put(remotename, stream);
}
public void Get(string remotename, System.IO.Stream stream)
{
m_openstack.Get(remotename, stream);
}
#endregion
#region IBackend implementation
public List<IFileEntry> List()
{
return m_openstack.List();
}
public void Put(string remotename, string filename)
{
m_openstack.Put(remotename, filename);
}
public void Get(string remotename, string filename)
{
m_openstack.Get(remotename, filename);
}
public void Delete(string remotename)
{
m_openstack.Delete(remotename);
}
public void Test()
{
m_openstack.Test();
}
public void CreateFolder()
{
m_openstack.CreateFolder();
}
public string DisplayName
{
get
{
return Strings.HubiC.DisplayName;
}
}
public string ProtocolKey
{
get
{
return "hubic";
}
}
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get {
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.HubiC.AuthidShort, Strings.HubiC.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("hubic"))),
});
}
}
public string Description
{
get
{
return Strings.HubiC.Description;
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_openstack != null)
{
m_openstack.Dispose();
m_openstack = null;
}
}
#endregion
}
+
public HubiCBackend(string url, Dictionary<string, string> options)
{
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
m_openstack = new OpenStackHelper(authid, url);
}
#region IStreamingBackend implementation
public void Put(string remotename, System.IO.Stream stream)
{
m_openstack.Put(remotename, stream);
}
public void Get(string remotename, System.IO.Stream stream)
{
m_openstack.Get(remotename, stream);
}
#endregion
#region IBackend implementation
public IEnumerable<IFileEntry> List()
{
return m_openstack.List();
}
public void Put(string remotename, string filename)
{
m_openstack.Put(remotename, filename);
}
public void Get(string remotename, string filename)
{
m_openstack.Get(remotename, filename);
}
public void Delete(string remotename)
{
m_openstack.Delete(remotename);
}
public void Test()
{
m_openstack.Test();
}
public void CreateFolder()
{
m_openstack.CreateFolder();
}
public string DisplayName
{
get
{
return Strings.HubiC.DisplayName;
}
}
public string ProtocolKey
{
get
{
return "hubic";
}
}
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get {
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.HubiC.AuthidShort, Strings.HubiC.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("hubic"))),
});
}
}
public string Description
{
get
{
return Strings.HubiC.Description;
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_openstack != null)
{
m_openstack.Dispose();
m_openstack = null;
}
}
#endregion
}
}
diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index 94a5dd16d..ba1bbd0d5 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -159,7 +159,7 @@ namespace Duplicati.Library.Backend get { return "jottacloud"; } } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { var doc = new System.Xml.XmlDocument(); try @@ -181,7 +181,6 @@ namespace Duplicati.Library.Backend // element must be a "folder", else it could also have been a "mountPoint" (which has a very similar structure). // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existant here. - List<IFileEntry> files = new List<IFileEntry>(); var xRoot = doc.DocumentElement; if (xRoot.Attributes["deleted"] != null) { @@ -192,7 +191,7 @@ namespace Duplicati.Library.Backend // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value); fe.IsFolder = true; - files.Add(fe); + yield return fe; } foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]")) { @@ -216,11 +215,10 @@ namespace Duplicati.Library.Backend if (xNode == null || !DateTime.TryParseExact(xNode.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out lastModified)) lastModified = new DateTime(); FileEntry fe = new FileEntry(name, size, lastModified, lastModified); - files.Add(fe); + yield return fe; } } } - return files; } public void Put(string remotename, string filename) @@ -263,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test() { - List(); + this.TestList(); } public void CreateFolder() diff --git a/Duplicati/Library/Backend/Mega/MegaBackend.cs b/Duplicati/Library/Backend/Mega/MegaBackend.cs index 732f0f040..e9261717f 100644 --- a/Duplicati/Library/Backend/Mega/MegaBackend.cs +++ b/Duplicati/Library/Backend/Mega/MegaBackend.cs @@ -170,16 +170,15 @@ namespace Duplicati.Library.Backend.Mega #region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
if (m_filecache == null)
ResetFileCache();
- return (
+ return
from n in m_filecache.Values
let item = n.OrderByDescending(x => x.ModificationDate).First()
- select (IFileEntry)new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0))
- ).ToList();
+ select new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0));
}
public void Put(string remotename, string filename)
@@ -218,7 +217,7 @@ namespace Duplicati.Library.Backend.Mega public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/OneDrive/OneDrive.cs b/Duplicati/Library/Backend/OneDrive/OneDrive.cs index 2b85409bb..30c31742a 100644 --- a/Duplicati/Library/Backend/OneDrive/OneDrive.cs +++ b/Duplicati/Library/Backend/OneDrive/OneDrive.cs @@ -212,7 +212,7 @@ namespace Duplicati.Library.Backend if (string.IsNullOrWhiteSpace(id))
{
// Refresh the list of files, just in case
- List();
+ foreach (IFileEntry file in List()) { /* We just need to iterate the whole list */ }
m_fileidCache.TryGetValue(name, out id);
if (string.IsNullOrWhiteSpace(id) && throwIfMissing)
@@ -232,7 +232,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -250,13 +250,11 @@ namespace Duplicati.Library.Backend get { return "onedrive"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
int offset = 0;
int count = FILE_LIST_PAGE_SIZE;
-
- var files = new List<IFileEntry>();
-
+
m_fileidCache.Clear();
while(count == FILE_LIST_PAGE_SIZE)
@@ -273,7 +271,7 @@ namespace Duplicati.Library.Backend var fe = new FileEntry(r.name, r.size.Value, r.updated_time.Value, r.updated_time.Value);
fe.IsFolder = string.Equals(r.type, "folder", StringComparison.OrdinalIgnoreCase);
- files.Add(fe);
+ yield return fe;
}
}
else
@@ -282,10 +280,7 @@ namespace Duplicati.Library.Backend }
offset += count;
-
}
-
- return files;
}
public void Put(string remotename, string filename)
diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 91b68f1cc..865306eb6 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -305,43 +305,15 @@ namespace Duplicati.Library.Backend.OpenStack }
#endregion
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var res = new List<IFileEntry>();
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
if (!string.IsNullOrEmpty(m_prefix))
plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix);
-
- var url = plainurl;
-
+
try
{
- while(true)
- {
- var req = m_helper.CreateRequest(url);
- req.Accept = "application/json";
-
- var items = m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req);
- foreach(var n in items)
- {
- var name = n.name;
- if (name.StartsWith(m_prefix))
- name = name.Substring(m_prefix.Length);
-
- if (n.bytes == null)
- res.Add(new FileEntry(name));
- else if (n.last_modified == null)
- res.Add(new FileEntry(name, n.bytes.Value));
- else
- res.Add(new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value));
- }
-
- if (items.Length != PAGE_LIMIT)
- return res;
-
- // Prepare next listing entry
- url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
- }
+ return ListWithoutExceptionCatch(plainurl);
}
catch(WebException wex)
{
@@ -351,6 +323,39 @@ namespace Duplicati.Library.Backend.OpenStack throw;
}
}
+
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch(string plainurl)
+ {
+ var url = plainurl;
+
+ while (true)
+ {
+ var req = m_helper.CreateRequest(url);
+ req.Accept = "application/json";
+
+ var items = m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req);
+ foreach (var n in items)
+ {
+ var name = n.name;
+ if (name.StartsWith(m_prefix))
+ name = name.Substring(m_prefix.Length);
+
+ if (n.bytes == null)
+ yield return new FileEntry(name);
+ else if (n.last_modified == null)
+ yield return new FileEntry(name, n.bytes.Value);
+ else
+ yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value);
+ }
+
+ if (items.Length != PAGE_LIMIT)
+ yield break;
+
+ // Prepare next listing entry
+ url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
+ }
+ }
+
public void Put(string remotename, string filename)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
@@ -369,7 +374,7 @@ namespace Duplicati.Library.Backend.OpenStack }
public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
{
diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 861a6dd69..c287fe491 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -291,20 +291,11 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- List<IFileEntry> lst = Connection.ListBucket(m_bucket, m_prefix);
- for (int i = 0; i < lst.Count; i++)
- {
- ((FileEntry)lst[i]).Name = lst[i].Name.Substring(m_prefix.Length);
-
- //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
- if (lst[i].Name.StartsWith("/") && !m_prefix.StartsWith("/"))
- ((FileEntry)lst[i]).Name = lst[i].Name.Substring(1);
- }
- return lst;
+ return ListWithouExceptionCatch();
}
catch (Exception ex)
{
@@ -317,6 +308,20 @@ namespace Duplicati.Library.Backend }
}
+ private IEnumerable<IFileEntry> ListWithouExceptionCatch()
+ {
+ foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix))
+ {
+ ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
+
+ //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
+ if (file.Name.StartsWith("/") && !m_prefix.StartsWith("/"))
+ ((FileEntry)file).Name = file.Name.Substring(1);
+
+ yield return file;
+ }
+ }
+
public void Put(string remotename, string localname)
{
using (System.IO.FileStream fs = System.IO.File.Open(localname, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
@@ -429,7 +434,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/S3/S3Wrapper.cs b/Duplicati/Library/Backend/S3/S3Wrapper.cs index cbf1f543b..667d6f5f9 100644 --- a/Duplicati/Library/Backend/S3/S3Wrapper.cs +++ b/Duplicati/Library/Backend/S3/S3Wrapper.cs @@ -135,12 +135,15 @@ namespace Duplicati.Library.Backend m_client.DeleteObject(objectDeleteRequest);
}
- public virtual List<IFileEntry> ListBucket(string bucketName, string prefix)
+ public virtual IEnumerable<IFileEntry> ListBucket(string bucketName, string prefix)
{
bool isTruncated = true;
string filename = null;
- List<IFileEntry> files = new List<IFileEntry>();
+ //TODO: Figure out if this is the case with AWSSDK too
+ //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
+ //So, track the files that have already been returned and skip any duplicates.
+ HashSet<string> alreadyReturned = new HashSet<string>();
//We truncate after ITEM_LIST_LIMIT elements, and then repeat
while (isTruncated)
@@ -161,29 +164,17 @@ namespace Duplicati.Library.Backend foreach (S3Object obj in listResponse.S3Objects)
{
- files.Add(new FileEntry(
- obj.Key,
- obj.Size,
- obj.LastModified,
- obj.LastModified
- ));
-
+ if (alreadyReturned.Add(obj.Key))
+ {
+ yield return new FileEntry(
+ obj.Key,
+ obj.Size,
+ obj.LastModified,
+ obj.LastModified
+ );
+ }
}
}
-
- //TODO: Figure out if this is the case with AWSSDK too
- //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
- Dictionary<string, string> tmp = new Dictionary<string, string>();
- for (int i = 0; i < files.Count; i++)
- if (tmp.ContainsKey(files[i].Name))
- {
- files.RemoveAt(i);
- i--;
- }
- else
- tmp.Add(files[i].Name, null);
-
- return files;
}
public void RenameFile(string bucketName, string source, string target)
diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 503ea4d80..1131c521f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -106,7 +106,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -313,10 +313,8 @@ namespace Duplicati.Library.Backend }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var files = new List<IFileEntry>();
-
string path = ".";
CreateConnection();
@@ -324,9 +322,7 @@ namespace Duplicati.Library.Backend foreach (Renci.SshNet.Sftp.SftpFile ls in m_con.ListDirectory(path))
if (ls.Name.ToString() != "." && ls.Name.ToString() != "..")
- files.Add(new FileEntry(ls.Name.ToString(), ls.Length, ls.LastAccessTime, ls.LastWriteTime) { IsFolder = ls.Attributes.IsDirectory });
-
- return files;
+ yield return new FileEntry(ls.Name.ToString(), ls.Length, ls.LastAccessTime, ls.LastWriteTime) { IsFolder = ls.Attributes.IsDirectory };
}
public static Renci.SshNet.PrivateKeyFile ValidateKeyFile(string filename, string password)
diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index 4793cf9a1..8e837e39f 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -393,8 +393,8 @@ namespace Duplicati.Library.Backend testContextForWeb(ctx, true); } - public List<IFileEntry> List() { return doList(false); } - private List<IFileEntry> doList(bool useNewContext) + public IEnumerable<IFileEntry> List() { return doList(false); } + private IEnumerable<IFileEntry> doList(bool useNewContext) { SP.ClientContext ctx = getSpClientContext(useNewContext); try @@ -407,20 +407,7 @@ namespace Duplicati.Library.Backend if (!remoteFolder.Exists) throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); - List<IFileEntry> files = new List<IFileEntry>(remoteFolder.Folders.Count + remoteFolder.Files.Count); - foreach (var f in remoteFolder.Folders.Where(ff => ff.Exists)) - { - FileEntry fe = new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated - fe.IsFolder = true; - files.Add(fe); - } - foreach (var f in remoteFolder.Files.Where(ff => ff.Exists)) - { - FileEntry fe = new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated - fe.IsFolder = false; - files.Add(fe); - } - return files; + return doListWithoutExceptionCatch(remoteFolder); } catch (ServerException) { throw; /* rethrow if Server answered */ } catch (Interface.FileMissingException) { throw; } @@ -429,6 +416,22 @@ namespace Duplicati.Library.Backend finally { } } + private IEnumerable<IFileEntry> doListWithoutExceptionCatch(SP.Folder remoteFolder) + { + foreach (var f in remoteFolder.Folders.Where(ff => ff.Exists)) + { + FileEntry fe = new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated + fe.IsFolder = true; + yield return fe; + } + foreach (var f in remoteFolder.Files.Where(ff => ff.Exists)) + { + FileEntry fe = new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated + fe.IsFolder = false; + yield return fe; + } + } + public void Get(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.Create(filename)) diff --git a/Duplicati/Library/Backend/Sia/Sia.cs b/Duplicati/Library/Backend/Sia/Sia.cs index a0ff731c0..2899d75aa 100644 --- a/Duplicati/Library/Backend/Sia/Sia.cs +++ b/Duplicati/Library/Backend/Sia/Sia.cs @@ -258,7 +258,7 @@ namespace Duplicati.Library.Backend.Sia public void Test() { - List(); + this.TestList(); } public void CreateFolder() @@ -277,15 +277,23 @@ namespace Duplicati.Library.Backend.Sia } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { - var files = new List<IFileEntry>(); try { SiaFileList fl = GetFiles(); - if (fl.Files == null) - return files; + return ListWithoutExceptionCatch(fl); + } + catch (System.Net.WebException wex) + { + throw new Exception("failed to call /renter/files "+wex.Message); + } + } + private IEnumerable<IFileEntry> ListWithoutExceptionCatch(SiaFileList fl) + { + if (fl.Files != null) + { foreach (var f in fl.Files) { // Sia returns a complete file list, but we're only interested in files that are @@ -295,16 +303,10 @@ namespace Duplicati.Library.Backend.Sia FileEntry fe = new FileEntry(f.Siapath.Substring(m_targetpath.Length + 1)); fe.Size = f.Filesize; fe.IsFolder = false; - files.Add(fe); + yield return fe; } } } - catch (System.Net.WebException wex) - { - throw new Exception("failed to call /renter/files "+wex.Message); - } - - return files; } public void Put(string remotename, string filename) diff --git a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs index 5d09b7c7a..2c78d77a9 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs +++ b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs @@ -123,7 +123,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -145,7 +145,7 @@ namespace Duplicati.Library.Backend get { return "tahoe"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
TahoeEl data;
@@ -184,7 +184,6 @@ namespace Duplicati.Library.Backend if (data == null || data.node == null || data.nodetype != "dirnode")
throw new Exception("Invalid folder listing response");
- var files = new List<IFileEntry>();
foreach (var e in data.node.children)
{
if (e.Value == null || e.Value.node == null)
@@ -205,10 +204,8 @@ namespace Duplicati.Library.Backend if (isFile)
fe.Size = e.Value.node.size;
- files.Add(fe);
+ yield return fe;
}
-
- return files;
}
public void Put(string remotename, string filename)
diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 6662755f8..d35815f16 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -125,126 +125,129 @@ namespace Duplicati.Library.Backend get { return "webdav"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var req = CreateRequest("");
+ return this.ListWithouExceptionCatch();
+ }
+ catch (System.Net.WebException wex)
+ {
+ if (wex.Response as System.Net.HttpWebResponse != null &&
+ ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict))
+ throw new Interface.FolderMissingException(Strings.WEBDAV.MissingFolderError(m_path, wex.Message), wex);
- req.Method = "PROPFIND";
- req.Headers.Add("Depth", "1");
- req.ContentType = "text/xml";
- req.ContentLength = PROPFIND_BODY.Length;
+ if (wex.Response as System.Net.HttpWebResponse != null && (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.MethodNotAllowed)
+ throw new UserInformationException(Strings.WEBDAV.MethodNotAllowedError((wex.Response as System.Net.HttpWebResponse).StatusCode), wex);
- var areq = new Utility.AsyncHttpRequest(req);
- using (System.IO.Stream s = areq.GetRequestStream())
- s.Write(PROPFIND_BODY, 0, PROPFIND_BODY.Length);
-
- var doc = new System.Xml.XmlDocument();
- using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
- {
- int code = (int)resp.StatusCode;
- if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
- throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
+ throw;
+ }
+ }
- if (!string.IsNullOrEmpty(m_debugPropfindFile))
- {
- using (var rs = areq.GetResponseStream())
- using (var fs = new System.IO.FileStream(m_debugPropfindFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
- Utility.Utility.CopyStream(rs, fs, false, m_copybuffer);
+ private IEnumerable<IFileEntry> ListWithouExceptionCatch()
+ {
+ var req = CreateRequest("");
- doc.Load(m_debugPropfindFile);
- }
- else
- {
- using (var rs = areq.GetResponseStream())
- doc.Load(rs);
- }
- }
+ req.Method = "PROPFIND";
+ req.Headers.Add("Depth", "1");
+ req.ContentType = "text/xml";
+ req.ContentLength = PROPFIND_BODY.Length;
- System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
- nm.AddNamespace("D", "DAV:");
+ var areq = new Utility.AsyncHttpRequest(req);
+ using (System.IO.Stream s = areq.GetRequestStream())
+ s.Write(PROPFIND_BODY, 0, PROPFIND_BODY.Length);
- List<IFileEntry> files = new List<IFileEntry>();
- m_filenamelist = new List<string>();
+ var doc = new System.Xml.XmlDocument();
+ using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
+ {
+ int code = (int)resp.StatusCode;
+ if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
+ throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
- foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
+ if (!string.IsNullOrEmpty(m_debugPropfindFile))
{
- //IIS uses %20 for spaces and %2B for +
- //Apache uses %20 for spaces and + for +
- string name = Library.Utility.Uri.UrlDecode(n.InnerText.Replace("+", "%2B"));
-
- string cmp_path;
-
- //TODO: This list is getting ridiculous, should change to regexps
-
- if (name.StartsWith(m_url))
- cmp_path = m_url;
- else if (name.StartsWith(m_rawurl))
- cmp_path = m_rawurl;
- else if (name.StartsWith(m_rawurlPort))
- cmp_path = m_rawurlPort;
- else if (name.StartsWith(m_path))
- cmp_path = m_path;
- else if (name.StartsWith("/" + m_path))
- cmp_path = "/" + m_path;
- else if (name.StartsWith(m_sanitizedUrl))
- cmp_path = m_sanitizedUrl;
- else if (name.StartsWith(m_reverseProtocolUrl))
- cmp_path = m_reverseProtocolUrl;
- else
- continue;
-
- if (name.Length <= cmp_path.Length)
- continue;
-
- name = name.Substring(cmp_path.Length);
-
- long size = -1;
- DateTime lastAccess = new DateTime();
- DateTime lastModified = new DateTime();
- bool isCollection = false;
-
- System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
- if (stat != null)
- {
- System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
- if (s != null)
- size = long.Parse(s.InnerText);
- s = stat.SelectSingleNode("D:getlastmodified", nm);
- if (s != null)
- try
- {
- //Not important if this succeeds
- lastAccess = lastModified = DateTime.Parse(s.InnerText, System.Globalization.CultureInfo.InvariantCulture);
- }
- catch { }
-
- s = stat.SelectSingleNode("D:iscollection", nm);
- if (s != null)
- isCollection = s.InnerText.Trim() == "1";
- else
- isCollection = (stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null);
- }
-
- FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
- fe.IsFolder = isCollection;
- files.Add(fe);
- m_filenamelist.Add(name);
- }
+ using (var rs = areq.GetResponseStream())
+ using (var fs = new System.IO.FileStream(m_debugPropfindFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
+ Utility.Utility.CopyStream(rs, fs, false, m_copybuffer);
- return files;
+ doc.Load(m_debugPropfindFile);
+ }
+ else
+ {
+ using (var rs = areq.GetResponseStream())
+ doc.Load(rs);
+ }
}
- catch (System.Net.WebException wex)
+
+ System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
+ nm.AddNamespace("D", "DAV:");
+
+ List<IFileEntry> files = new List<IFileEntry>();
+ m_filenamelist = new List<string>();
+
+ foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
{
- if (wex.Response as System.Net.HttpWebResponse != null &&
- ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict))
- throw new Interface.FolderMissingException(Strings.WEBDAV.MissingFolderError(m_path, wex.Message), wex);
+ //IIS uses %20 for spaces and %2B for +
+ //Apache uses %20 for spaces and + for +
+ string name = Library.Utility.Uri.UrlDecode(n.InnerText.Replace("+", "%2B"));
+
+ string cmp_path;
+
+ //TODO: This list is getting ridiculous, should change to regexps
+
+ if (name.StartsWith(m_url))
+ cmp_path = m_url;
+ else if (name.StartsWith(m_rawurl))
+ cmp_path = m_rawurl;
+ else if (name.StartsWith(m_rawurlPort))
+ cmp_path = m_rawurlPort;
+ else if (name.StartsWith(m_path))
+ cmp_path = m_path;
+ else if (name.StartsWith("/" + m_path))
+ cmp_path = "/" + m_path;
+ else if (name.StartsWith(m_sanitizedUrl))
+ cmp_path = m_sanitizedUrl;
+ else if (name.StartsWith(m_reverseProtocolUrl))
+ cmp_path = m_reverseProtocolUrl;
+ else
+ continue;
- if (wex.Response as System.Net.HttpWebResponse != null && (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.MethodNotAllowed)
- throw new UserInformationException(Strings.WEBDAV.MethodNotAllowedError((wex.Response as System.Net.HttpWebResponse).StatusCode), wex);
+ if (name.Length <= cmp_path.Length)
+ continue;
+
+ name = name.Substring(cmp_path.Length);
+
+ long size = -1;
+ DateTime lastAccess = new DateTime();
+ DateTime lastModified = new DateTime();
+ bool isCollection = false;
+
+ System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
+ if (stat != null)
+ {
+ System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
+ if (s != null)
+ size = long.Parse(s.InnerText);
+ s = stat.SelectSingleNode("D:getlastmodified", nm);
+ if (s != null)
+ try
+ {
+ //Not important if this succeeds
+ lastAccess = lastModified = DateTime.Parse(s.InnerText, System.Globalization.CultureInfo.InvariantCulture);
+ }
+ catch { }
+
+ s = stat.SelectSingleNode("D:iscollection", nm);
+ if (s != null)
+ isCollection = s.InnerText.Trim() == "1";
+ else
+ isCollection = (stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null);
+ }
- throw;
+ FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
+ fe.IsFolder = isCollection;
+ m_filenamelist.Add(name);
+ yield return fe;
}
}
@@ -308,7 +311,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
|