diff options
Diffstat (limited to 'Duplicati')
32 files changed, 408 insertions, 380 deletions
diff --git a/Duplicati/CommandLine/BackendTester/Program.cs b/Duplicati/CommandLine/BackendTester/Program.cs index 142bf0e0f..47a578d3e 100644 --- a/Duplicati/CommandLine/BackendTester/Program.cs +++ b/Duplicati/CommandLine/BackendTester/Program.cs @@ -161,7 +161,7 @@ namespace Duplicati.CommandLine.BackendTester try
{
- List<Library.Interface.IFileEntry> curlist = null;
+ IEnumerable<Library.Interface.IFileEntry> curlist = null;
try
{
curlist = backend.List();
diff --git a/Duplicati/CommandLine/RecoveryTool/Download.cs b/Duplicati/CommandLine/RecoveryTool/Download.cs index e15635224..2a22a36ac 100644 --- a/Duplicati/CommandLine/RecoveryTool/Download.cs +++ b/Duplicati/CommandLine/RecoveryTool/Download.cs @@ -17,6 +17,7 @@ using System;
using System.IO;
using System.Collections.Generic;
+using System.Linq;
namespace Duplicati.CommandLine.RecoveryTool
{
@@ -48,7 +49,7 @@ namespace Duplicati.CommandLine.RecoveryTool Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey);
- var lst = backend.List();
+ var lst = backend.List().ToList();
Console.WriteLine("Found {0} files", lst.Count);
diff --git a/Duplicati/CommandLine/RecoveryTool/Recompress.cs b/Duplicati/CommandLine/RecoveryTool/Recompress.cs index 95a5eab42..21dddd731 100644 --- a/Duplicati/CommandLine/RecoveryTool/Recompress.cs +++ b/Duplicati/CommandLine/RecoveryTool/Recompress.cs @@ -47,7 +47,7 @@ namespace Duplicati.CommandLine.RecoveryTool Console.WriteLine("Listing files on backend: {0} ...", backend.ProtocolKey); - var rawlist = backend.List(); + var rawlist = backend.List().ToList(); Console.WriteLine("Found {0} files at remote storage", rawlist.Count); diff --git a/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs b/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs index 9cdcc09b9..032854043 100644 --- a/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs +++ b/Duplicati/Library/Backend/AlternativeFTP/AlternativeFTPBackend.cs @@ -194,17 +194,17 @@ namespace Duplicati.Library.Backend.AlternativeFTP }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
return List("");
}
- public List<IFileEntry> List(string filename)
+ public IEnumerable<IFileEntry> List(string filename)
{
return List(filename, false);
}
- private List<IFileEntry> List(string filename, bool stripFile)
+ private IEnumerable<IFileEntry> List(string filename, bool stripFile)
{
var list = new List<IFileEntry>();
string remotePath = filename;
diff --git a/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs b/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs index e24a949fa..b68319333 100644 --- a/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs +++ b/Duplicati/Library/Backend/AmazonCloudDrive/AmzCD.cs @@ -336,7 +336,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive #endregion
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
EnforceConsistencyDelay(RemoteOperation.List);
@@ -416,7 +416,7 @@ namespace Duplicati.Library.Backend.AmazonCloudDrive }
public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
{
diff --git a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs index 2e200c8fd..d55eb17b0 100644 --- a/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs +++ b/Duplicati/Library/Backend/AzureBlob/AzureBlobBackend.cs @@ -82,7 +82,7 @@ namespace Duplicati.Library.Backend.AzureBlob get { return true; } } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { return _azureBlob.ListContainerEntries(); } @@ -161,7 +161,7 @@ namespace Duplicati.Library.Backend.AzureBlob public void Test() { - List(); + this.TestList(); } public void CreateFolder() diff --git a/Duplicati/Library/Backend/Backblaze/B2.cs b/Duplicati/Library/Backend/Backblaze/B2.cs index 383e2e009..682e05e23 100644 --- a/Duplicati/Library/Backend/Backblaze/B2.cs +++ b/Duplicati/Library/Backend/Backblaze/B2.cs @@ -285,7 +285,7 @@ namespace Duplicati.Library.Backend.Backblaze }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
m_filecache = null;
var cache = new Dictionary<string, List<FileEntity>>();
@@ -380,7 +380,7 @@ namespace Duplicati.Library.Backend.Backblaze public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/Box/BoxBackend.cs b/Duplicati/Library/Backend/Box/BoxBackend.cs index 51e3c838e..124a51fe7 100644 --- a/Duplicati/Library/Backend/Box/BoxBackend.cs +++ b/Duplicati/Library/Backend/Box/BoxBackend.cs @@ -252,12 +252,11 @@ namespace Duplicati.Library.Backend.Box #region IBackend implementation
- public System.Collections.Generic.List<IFileEntry> List()
+ public System.Collections.Generic.IEnumerable<IFileEntry> List()
{
- return (
+ return
from n in PagedFileListResponse(CurrentFolder, false)
- select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" }
- ).ToList();
+ select (IFileEntry)new FileEntry(n.Name, n.Size, n.ModifiedAt, n.ModifiedAt) { IsFolder = n.Type == "folder" };
}
public void Put(string remotename, string filename)
@@ -295,7 +294,7 @@ namespace Duplicati.Library.Backend.Box public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs index 1b33756d3..29cbc2ba1 100644 --- a/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs +++ b/Duplicati/Library/Backend/CloudFiles/CloudFiles.cs @@ -19,6 +19,7 @@ #endregion
using System;
using System.Collections.Generic;
+using System.Linq;
using System.Text;
using System.Net;
using Duplicati.Library.Interface;
@@ -128,9 +129,8 @@ namespace Duplicati.Library.Backend get { return "cloudfiles"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var files = new List<IFileEntry>();
string extraUrl = "?format=xml&limit=" + ITEM_LIST_LIMIT.ToString();
string markerUrl = "";
@@ -182,7 +182,7 @@ namespace Duplicati.Library.Backend mod = new DateTime();
lastItemName = name;
- files.Add(new FileEntry(name, size, mod, mod));
+ yield return new FileEntry(name, size, mod, mod);
}
repeat = lst.Count == ITEM_LIST_LIMIT;
@@ -191,8 +191,6 @@ namespace Duplicati.Library.Backend markerUrl = "&marker=" + Library.Utility.Uri.UrlEncode(lastItemName);
} while (repeat);
-
- return files;
}
public void Put(string remotename, string filename)
@@ -263,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test()
{
//The "Folder not found" is not detectable :(
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/Dropbox/Dropbox.cs b/Duplicati/Library/Backend/Dropbox/Dropbox.cs index 8a70684a0..76e555c0a 100644 --- a/Duplicati/Library/Backend/Dropbox/Dropbox.cs +++ b/Duplicati/Library/Backend/Dropbox/Dropbox.cs @@ -67,35 +67,37 @@ namespace Duplicati.Library.Backend return ife;
}
-
- public List<IFileEntry> List()
+
+ public IEnumerable<IFileEntry> List()
{
try
{
- var list = new List<IFileEntry>();
- var lfr = dbx.ListFiles(m_path);
-
- foreach (var md in lfr.entries)
- list.Add(ParseEntry(md));
-
- while (lfr.has_more)
- {
- lfr = dbx.ListFilesContinue(lfr.cursor);
- foreach (var md in lfr.entries)
- list.Add(ParseEntry(md));
- }
-
- return list;
+ return ListWithoutExceptionCatch();
}
catch (DropboxException de)
{
if (de.errorJSON["error"][".tag"].ToString() == "path" && de.errorJSON["error"]["path"][".tag"].ToString() == "not_found")
throw new FolderMissingException();
-
+
throw;
}
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ var lfr = dbx.ListFiles(m_path);
+
+ foreach (var md in lfr.entries)
+ yield return ParseEntry(md);
+
+ while (lfr.has_more)
+ {
+ lfr = dbx.ListFilesContinue(lfr.cursor);
+ foreach (var md in lfr.entries)
+ yield return ParseEntry(md);
+ }
+ }
+
public void Put(string remotename, string filename)
{
using(FileStream fs = System.IO.File.OpenRead(filename))
@@ -136,7 +138,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/FTP/FTPBackend.cs b/Duplicati/Library/Backend/FTP/FTPBackend.cs index 1aea8537d..6ac888979 100644 --- a/Duplicati/Library/Backend/FTP/FTPBackend.cs +++ b/Duplicati/Library/Backend/FTP/FTPBackend.cs @@ -170,12 +170,12 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
return List("");
}
-
- public List<IFileEntry> List(string filename)
+
+ public IEnumerable<IFileEntry> List(string filename)
{
var req = CreateRequest(filename);
req.Method = System.Net.WebRequestMethods.Ftp.ListDirectoryDetails;
@@ -183,21 +183,7 @@ namespace Duplicati.Library.Backend try
{
- var lst = new List<IFileEntry>();
- var areq = new Utility.AsyncHttpRequest(req);
- using (var resp = areq.GetResponse())
- using (var rs = areq.GetResponseStream())
- using (var sr = new System.IO.StreamReader(new StreamReadHelper(rs)))
- {
- string line;
- while ((line = sr.ReadLine()) != null)
- {
- FileEntry f = ParseLine(line);
- if (f != null)
- lst.Add(f);
- }
- }
- return lst;
+ return ListWithoutExceptionCatch(req);
}
catch (System.Net.WebException wex)
{
@@ -208,6 +194,23 @@ namespace Duplicati.Library.Backend }
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch(System.Net.FtpWebRequest req)
+ {
+ var areq = new Utility.AsyncHttpRequest(req);
+ using (var resp = areq.GetResponse())
+ using (var rs = areq.GetResponseStream())
+ using (var sr = new System.IO.StreamReader(new StreamReadHelper(rs)))
+ {
+ string line;
+ while ((line = sr.ReadLine()) != null)
+ {
+ FileEntry f = ParseLine(line);
+ if (f != null)
+ yield return f;
+ }
+ }
+ }
+
public void Put(string remotename, System.IO.Stream input)
{
System.Net.FtpWebRequest req = null;
@@ -227,7 +230,7 @@ namespace Duplicati.Library.Backend if (m_listVerify)
{
- List<IFileEntry> files = List(remotename);
+ IEnumerable<IFileEntry> files = List(remotename);
foreach(IFileEntry fe in files)
if (fe.Name.Equals(remotename) || fe.Name.EndsWith("/" + remotename) || fe.Name.EndsWith("\\" + remotename))
{
@@ -308,7 +311,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/File/FileBackend.cs b/Duplicati/Library/Backend/File/FileBackend.cs index cafba1170..8a846594b 100644 --- a/Duplicati/Library/Backend/File/FileBackend.cs +++ b/Duplicati/Library/Backend/File/FileBackend.cs @@ -164,30 +164,26 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- List<IFileEntry> ls = new List<IFileEntry>();
-
PreAuthenticate();
if (!System.IO.Directory.Exists(m_path))
throw new FolderMissingException(Strings.FileBackend.FolderMissingError(m_path));
- foreach (string s in System.IO.Directory.GetFiles(m_path))
+ foreach (string s in System.IO.Directory.EnumerateFiles(m_path))
{
System.IO.FileInfo fi = new System.IO.FileInfo(s);
- ls.Add(new FileEntry(fi.Name, fi.Length, fi.LastAccessTime, fi.LastWriteTime));
+ yield return new FileEntry(fi.Name, fi.Length, fi.LastAccessTime, fi.LastWriteTime);
}
- foreach (string s in System.IO.Directory.GetDirectories(m_path))
+ foreach (string s in System.IO.Directory.EnumerateDirectories(m_path))
{
System.IO.DirectoryInfo di = new System.IO.DirectoryInfo(s);
FileEntry fe = new FileEntry(di.Name, 0, di.LastAccessTime, di.LastWriteTime);
fe.IsFolder = true;
- ls.Add(fe);
+ yield return fe;
}
-
- return ls;
}
#if DEBUG_RETRY
@@ -265,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs index dcdeb64db..663bd274b 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleCloudStorage.cs @@ -98,8 +98,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage m_oauth = new OAuthHelper(authid, this.ProtocolKey);
m_oauth.AutoAuthHeader = true;
}
-
-
+
private class ListBucketResponse
{
@@ -130,39 +129,13 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public string location { get; set; }
public string storageClass { get; set; }
}
+
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var res = new List<IFileEntry>();
- string token = null;
- do
- {
- var url = string.Format("{0}/b/{1}/o?prefix={2}", API_URL, m_bucket, Library.Utility.Uri.UrlEncode(m_prefix));
- if (!string.IsNullOrEmpty(token))
- url += string.Format("&pageToken={0}", token);
- var resp = m_oauth.ReadJSONResponse<ListBucketResponse>(url);
-
- if (resp.items != null)
- foreach(var f in resp.items)
- {
- var name = f.name;
- if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
- name = name.Substring(m_prefix.Length);
- if (f.size == null)
- res.Add(new FileEntry(name));
- else if (f.updated == null)
- res.Add(new FileEntry(name, f.size.Value));
- else
- res.Add(new FileEntry(name, f.size.Value, f.updated.Value, f.updated.Value));
- }
-
- token = resp.nextPageToken;
-
- } while(!string.IsNullOrEmpty(token));
-
- return res;
+ return this.ListWithoutExceptionCatch();
}
catch (WebException wex)
{
@@ -173,6 +146,35 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage }
}
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ string token = null;
+ do
+ {
+ var url = string.Format("{0}/b/{1}/o?prefix={2}", API_URL, m_bucket, Library.Utility.Uri.UrlEncode(m_prefix));
+ if (!string.IsNullOrEmpty(token))
+ url += string.Format("&pageToken={0}", token);
+ var resp = m_oauth.ReadJSONResponse<ListBucketResponse>(url);
+
+ if (resp.items != null)
+ foreach (var f in resp.items)
+ {
+ var name = f.name;
+ if (name.StartsWith(m_prefix, StringComparison.OrdinalIgnoreCase))
+ name = name.Substring(m_prefix.Length);
+ if (f.size == null)
+ yield return new FileEntry(name);
+ else if (f.updated == null)
+ yield return new FileEntry(name, f.size.Value);
+ else
+ yield return new FileEntry(name, f.size.Value, f.updated.Value, f.updated.Value);
+ }
+
+ token = resp.nextPageToken;
+
+ } while (!string.IsNullOrEmpty(token));
+ }
+
public void Put(string remotename, string filename)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
@@ -195,7 +197,7 @@ namespace Duplicati.Library.Backend.GoogleCloudStorage public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs index c4e6af992..736044bbe 100644 --- a/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs +++ b/Duplicati/Library/Backend/GoogleServices/GoogleDrive.cs @@ -189,44 +189,14 @@ namespace Duplicati.Library.Backend.GoogleDrive #region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var res = new List<IFileEntry>();
m_filecache.Clear();
- foreach(var n in ListFolder(CurrentFolderId))
- {
- FileEntry fe = null;
-
- if (n.fileSize == null)
- fe = new FileEntry(n.title);
- else if (n.modifiedDate == null)
- fe = new FileEntry(n.title, n.fileSize.Value);
- else
- fe = new FileEntry(n.title, n.fileSize.Value, n.modifiedDate.Value, n.modifiedDate.Value);
-
- if (fe != null)
- {
- fe.IsFolder = FOLDER_MIMETYPE.Equals(n.mimeType, StringComparison.OrdinalIgnoreCase);
- res.Add(fe);
-
- if (!fe.IsFolder)
- {
- GoogleDriveFolderItem[] lst;
- if (!m_filecache.TryGetValue(fe.Name, out lst))
- m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
- else
- {
- Array.Resize(ref lst, lst.Length + 1);
- lst[lst.Length - 1] = n;
- }
- }
- }
- }
-
- return res;
+ // For now, this class assumes that List() fully populates the file cache
+ return ListWithoutExceptionCatch().ToList();
}
catch
{
@@ -234,7 +204,42 @@ namespace Duplicati.Library.Backend.GoogleDrive throw;
}
+ }
+
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch()
+ {
+ foreach (var n in ListFolder(CurrentFolderId))
+ {
+ FileEntry fe = null;
+
+ if (n.fileSize == null)
+ fe = new FileEntry(n.title);
+ else if (n.modifiedDate == null)
+ fe = new FileEntry(n.title, n.fileSize.Value);
+ else
+ fe = new FileEntry(n.title, n.fileSize.Value, n.modifiedDate.Value, n.modifiedDate.Value);
+
+ if (fe != null)
+ {
+ fe.IsFolder = FOLDER_MIMETYPE.Equals(n.mimeType, StringComparison.OrdinalIgnoreCase);
+
+ if (!fe.IsFolder)
+ {
+ GoogleDriveFolderItem[] lst;
+ if (!m_filecache.TryGetValue(fe.Name, out lst))
+ {
+ m_filecache[fe.Name] = new GoogleDriveFolderItem[] { n };
+ }
+ else
+ {
+ Array.Resize(ref lst, lst.Length + 1);
+ lst[lst.Length - 1] = n;
+ }
+ }
+ yield return fe;
+ }
+ }
}
public void Put(string remotename, string filename)
@@ -274,7 +279,7 @@ namespace Duplicati.Library.Backend.GoogleDrive public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/HubiC/HubiCBackend.cs b/Duplicati/Library/Backend/HubiC/HubiCBackend.cs index 748842d59..1f8fb03c0 100644 --- a/Duplicati/Library/Backend/HubiC/HubiCBackend.cs +++ b/Duplicati/Library/Backend/HubiC/HubiCBackend.cs @@ -21,6 +21,6 @@ namespace Duplicati.Library.Backend.HubiC public class HubiCBackend : IBackend, IStreamingBackend
{
private const string AUTHID_OPTION = "authid";
private const string HUBIC_API_URL = "https://api.hubic.com/1.0/";
private const string HUBIC_API_CREDENTIAL_URL = HUBIC_API_URL + "account/credentials";
private OpenStackStorage m_openstack;
private class HubiCAuthResponse
{
public string token { get; set; }
public string endpoint { get; set; }
public DateTime? expires { get; set;}
}
private class OpenStackHelper : OpenStackStorage
{
private OAuthHelper m_helper;
private HubiCAuthResponse m_token;
public OpenStackHelper(string authid, string url)
: base(url, MockOptions())
{
m_helper = new OAuthHelper(authid, "hubic") { AutoAuthHeader = true };
}
private static Dictionary<string, string> MockOptions()
{
var res = new Dictionary<string, string>();
res["openstack-authuri"] = "invalid://dont-use";
res["openstack-apikey"] = "invalid";
res["auth-username"] = "invalid";
return res;
}
private HubiCAuthResponse AuthToken
{
get
{
if (m_token == null || (m_token.expires != null && (m_token.expires.Value - DateTime.UtcNow).TotalSeconds < 30))
m_token = m_helper.ReadJSONResponse<HubiCAuthResponse>(HUBIC_API_CREDENTIAL_URL);
return m_token;
}
}
protected override string AccessToken
{
get { return AuthToken.token; }
}
protected override string SimpleStorageEndPoint
{
get { return AuthToken.endpoint; }
}
}
public HubiCBackend()
{
}
-
public HubiCBackend(string url, Dictionary<string, string> options)
{
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
m_openstack = new OpenStackHelper(authid, url);
}
#region IStreamingBackend implementation
public void Put(string remotename, System.IO.Stream stream)
{
m_openstack.Put(remotename, stream);
}
public void Get(string remotename, System.IO.Stream stream)
{
m_openstack.Get(remotename, stream);
}
#endregion
#region IBackend implementation
public List<IFileEntry> List()
{
return m_openstack.List();
}
public void Put(string remotename, string filename)
{
m_openstack.Put(remotename, filename);
}
public void Get(string remotename, string filename)
{
m_openstack.Get(remotename, filename);
}
public void Delete(string remotename)
{
m_openstack.Delete(remotename);
}
public void Test()
{
m_openstack.Test();
}
public void CreateFolder()
{
m_openstack.CreateFolder();
}
public string DisplayName
{
get
{
return Strings.HubiC.DisplayName;
}
}
public string ProtocolKey
{
get
{
return "hubic";
}
}
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get {
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.HubiC.AuthidShort, Strings.HubiC.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("hubic"))),
});
}
}
public string Description
{
get
{
return Strings.HubiC.Description;
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_openstack != null)
{
m_openstack.Dispose();
m_openstack = null;
}
}
#endregion
}
+
public HubiCBackend(string url, Dictionary<string, string> options)
{
string authid = null;
if (options.ContainsKey(AUTHID_OPTION))
authid = options[AUTHID_OPTION];
m_openstack = new OpenStackHelper(authid, url);
}
#region IStreamingBackend implementation
public void Put(string remotename, System.IO.Stream stream)
{
m_openstack.Put(remotename, stream);
}
public void Get(string remotename, System.IO.Stream stream)
{
m_openstack.Get(remotename, stream);
}
#endregion
#region IBackend implementation
public IEnumerable<IFileEntry> List()
{
return m_openstack.List();
}
public void Put(string remotename, string filename)
{
m_openstack.Put(remotename, filename);
}
public void Get(string remotename, string filename)
{
m_openstack.Get(remotename, filename);
}
public void Delete(string remotename)
{
m_openstack.Delete(remotename);
}
public void Test()
{
m_openstack.Test();
}
public void CreateFolder()
{
m_openstack.CreateFolder();
}
public string DisplayName
{
get
{
return Strings.HubiC.DisplayName;
}
}
public string ProtocolKey
{
get
{
return "hubic";
}
}
public System.Collections.Generic.IList<ICommandLineArgument> SupportedCommands
{
get {
return new List<ICommandLineArgument>(new ICommandLineArgument[] {
new CommandLineArgument(AUTHID_OPTION, CommandLineArgument.ArgumentType.Password, Strings.HubiC.AuthidShort, Strings.HubiC.AuthidLong(OAuthHelper.OAUTH_LOGIN_URL("hubic"))),
});
}
}
public string Description
{
get
{
return Strings.HubiC.Description;
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_openstack != null)
{
m_openstack.Dispose();
m_openstack = null;
}
}
#endregion
}
}
diff --git a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs index 94a5dd16d..ba1bbd0d5 100644 --- a/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs +++ b/Duplicati/Library/Backend/Jottacloud/Jottacloud.cs @@ -159,7 +159,7 @@ namespace Duplicati.Library.Backend get { return "jottacloud"; } } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { var doc = new System.Xml.XmlDocument(); try @@ -181,7 +181,6 @@ namespace Duplicati.Library.Backend // element must be a "folder", else it could also have been a "mountPoint" (which has a very similar structure). // We must check for "deleted" attribute, because files/folders which has it is deleted (attribute contains the timestamp of deletion) // so we treat them as non-existant here. - List<IFileEntry> files = new List<IFileEntry>(); var xRoot = doc.DocumentElement; if (xRoot.Attributes["deleted"] != null) { @@ -192,7 +191,7 @@ namespace Duplicati.Library.Backend // Subfolders are only listed with name. We can get a timestamp by sending a request for each folder, but that is probably not necessary? FileEntry fe = new FileEntry(xFolder.Attributes["name"].Value); fe.IsFolder = true; - files.Add(fe); + yield return fe; } foreach (System.Xml.XmlNode xFile in xRoot.SelectNodes("files/file[not(@deleted)]")) { @@ -216,11 +215,10 @@ namespace Duplicati.Library.Backend if (xNode == null || !DateTime.TryParseExact(xNode.InnerText, JFS_DATE_FORMAT, System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AdjustToUniversal, out lastModified)) lastModified = new DateTime(); FileEntry fe = new FileEntry(name, size, lastModified, lastModified); - files.Add(fe); + yield return fe; } } } - return files; } public void Put(string remotename, string filename) @@ -263,7 +261,7 @@ namespace Duplicati.Library.Backend public void Test() { - List(); + this.TestList(); } public void CreateFolder() diff --git a/Duplicati/Library/Backend/Mega/MegaBackend.cs b/Duplicati/Library/Backend/Mega/MegaBackend.cs index 732f0f040..e9261717f 100644 --- a/Duplicati/Library/Backend/Mega/MegaBackend.cs +++ b/Duplicati/Library/Backend/Mega/MegaBackend.cs @@ -170,16 +170,15 @@ namespace Duplicati.Library.Backend.Mega #region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
if (m_filecache == null)
ResetFileCache();
- return (
+ return
from n in m_filecache.Values
let item = n.OrderByDescending(x => x.ModificationDate).First()
- select (IFileEntry)new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0))
- ).ToList();
+ select new FileEntry(item.Name, item.Size, item.ModificationDate ?? new DateTime(0), item.ModificationDate ?? new DateTime(0));
}
public void Put(string remotename, string filename)
@@ -218,7 +217,7 @@ namespace Duplicati.Library.Backend.Mega public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/OneDrive/OneDrive.cs b/Duplicati/Library/Backend/OneDrive/OneDrive.cs index 2b85409bb..30c31742a 100644 --- a/Duplicati/Library/Backend/OneDrive/OneDrive.cs +++ b/Duplicati/Library/Backend/OneDrive/OneDrive.cs @@ -212,7 +212,7 @@ namespace Duplicati.Library.Backend if (string.IsNullOrWhiteSpace(id))
{
// Refresh the list of files, just in case
- List();
+ foreach (IFileEntry file in List()) { /* We just need to iterate the whole list */ }
m_fileidCache.TryGetValue(name, out id);
if (string.IsNullOrWhiteSpace(id) && throwIfMissing)
@@ -232,7 +232,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -250,13 +250,11 @@ namespace Duplicati.Library.Backend get { return "onedrive"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
int offset = 0;
int count = FILE_LIST_PAGE_SIZE;
-
- var files = new List<IFileEntry>();
-
+
m_fileidCache.Clear();
while(count == FILE_LIST_PAGE_SIZE)
@@ -273,7 +271,7 @@ namespace Duplicati.Library.Backend var fe = new FileEntry(r.name, r.size.Value, r.updated_time.Value, r.updated_time.Value);
fe.IsFolder = string.Equals(r.type, "folder", StringComparison.OrdinalIgnoreCase);
- files.Add(fe);
+ yield return fe;
}
}
else
@@ -282,10 +280,7 @@ namespace Duplicati.Library.Backend }
offset += count;
-
}
-
- return files;
}
public void Put(string remotename, string filename)
diff --git a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs index 91b68f1cc..865306eb6 100644 --- a/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs +++ b/Duplicati/Library/Backend/OpenStack/OpenStackStorage.cs @@ -305,43 +305,15 @@ namespace Duplicati.Library.Backend.OpenStack }
#endregion
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var res = new List<IFileEntry>();
var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT);
if (!string.IsNullOrEmpty(m_prefix))
plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix);
-
- var url = plainurl;
-
+
try
{
- while(true)
- {
- var req = m_helper.CreateRequest(url);
- req.Accept = "application/json";
-
- var items = m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req);
- foreach(var n in items)
- {
- var name = n.name;
- if (name.StartsWith(m_prefix))
- name = name.Substring(m_prefix.Length);
-
- if (n.bytes == null)
- res.Add(new FileEntry(name));
- else if (n.last_modified == null)
- res.Add(new FileEntry(name, n.bytes.Value));
- else
- res.Add(new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value));
- }
-
- if (items.Length != PAGE_LIMIT)
- return res;
-
- // Prepare next listing entry
- url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
- }
+ return ListWithoutExceptionCatch(plainurl);
}
catch(WebException wex)
{
@@ -351,6 +323,39 @@ namespace Duplicati.Library.Backend.OpenStack throw;
}
}
+
+ private IEnumerable<IFileEntry> ListWithoutExceptionCatch(string plainurl)
+ {
+ var url = plainurl;
+
+ while (true)
+ {
+ var req = m_helper.CreateRequest(url);
+ req.Accept = "application/json";
+
+ var items = m_helper.ReadJSONResponse<OpenStackStorageItem[]>(req);
+ foreach (var n in items)
+ {
+ var name = n.name;
+ if (name.StartsWith(m_prefix))
+ name = name.Substring(m_prefix.Length);
+
+ if (n.bytes == null)
+ yield return new FileEntry(name);
+ else if (n.last_modified == null)
+ yield return new FileEntry(name, n.bytes.Value);
+ else
+ yield return new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value);
+ }
+
+ if (items.Length != PAGE_LIMIT)
+ yield break;
+
+ // Prepare next listing entry
+ url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name));
+ }
+ }
+
public void Put(string remotename, string filename)
{
using (System.IO.FileStream fs = System.IO.File.OpenRead(filename))
@@ -369,7 +374,7 @@ namespace Duplicati.Library.Backend.OpenStack }
public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
{
diff --git a/Duplicati/Library/Backend/S3/S3Backend.cs b/Duplicati/Library/Backend/S3/S3Backend.cs index 861a6dd69..c287fe491 100644 --- a/Duplicati/Library/Backend/S3/S3Backend.cs +++ b/Duplicati/Library/Backend/S3/S3Backend.cs @@ -291,20 +291,11 @@ namespace Duplicati.Library.Backend get { return true; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- List<IFileEntry> lst = Connection.ListBucket(m_bucket, m_prefix);
- for (int i = 0; i < lst.Count; i++)
- {
- ((FileEntry)lst[i]).Name = lst[i].Name.Substring(m_prefix.Length);
-
- //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
- if (lst[i].Name.StartsWith("/") && !m_prefix.StartsWith("/"))
- ((FileEntry)lst[i]).Name = lst[i].Name.Substring(1);
- }
- return lst;
+ return ListWithouExceptionCatch();
}
catch (Exception ex)
{
@@ -317,6 +308,20 @@ namespace Duplicati.Library.Backend }
}
+ private IEnumerable<IFileEntry> ListWithouExceptionCatch()
+ {
+ foreach (IFileEntry file in Connection.ListBucket(m_bucket, m_prefix))
+ {
+ ((FileEntry)file).Name = file.Name.Substring(m_prefix.Length);
+
+ //Fix for a bug in Duplicati 1.0 beta 3 and earlier, where filenames are incorrectly prefixed with a slash
+ if (file.Name.StartsWith("/") && !m_prefix.StartsWith("/"))
+ ((FileEntry)file).Name = file.Name.Substring(1);
+
+ yield return file;
+ }
+ }
+
public void Put(string remotename, string localname)
{
using (System.IO.FileStream fs = System.IO.File.Open(localname, System.IO.FileMode.Open, System.IO.FileAccess.Read, System.IO.FileShare.Read))
@@ -429,7 +434,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Backend/S3/S3Wrapper.cs b/Duplicati/Library/Backend/S3/S3Wrapper.cs index cbf1f543b..667d6f5f9 100644 --- a/Duplicati/Library/Backend/S3/S3Wrapper.cs +++ b/Duplicati/Library/Backend/S3/S3Wrapper.cs @@ -135,12 +135,15 @@ namespace Duplicati.Library.Backend m_client.DeleteObject(objectDeleteRequest);
}
- public virtual List<IFileEntry> ListBucket(string bucketName, string prefix)
+ public virtual IEnumerable<IFileEntry> ListBucket(string bucketName, string prefix)
{
bool isTruncated = true;
string filename = null;
- List<IFileEntry> files = new List<IFileEntry>();
+ //TODO: Figure out if this is the case with AWSSDK too
+ //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
+ //So, track the files that have already been returned and skip any duplicates.
+ HashSet<string> alreadyReturned = new HashSet<string>();
//We truncate after ITEM_LIST_LIMIT elements, and then repeat
while (isTruncated)
@@ -161,29 +164,17 @@ namespace Duplicati.Library.Backend foreach (S3Object obj in listResponse.S3Objects)
{
- files.Add(new FileEntry(
- obj.Key,
- obj.Size,
- obj.LastModified,
- obj.LastModified
- ));
-
+ if (alreadyReturned.Add(obj.Key))
+ {
+ yield return new FileEntry(
+ obj.Key,
+ obj.Size,
+ obj.LastModified,
+ obj.LastModified
+ );
+ }
}
}
-
- //TODO: Figure out if this is the case with AWSSDK too
- //Unfortunately S3 sometimes reports duplicate values when requesting more than one page of results
- Dictionary<string, string> tmp = new Dictionary<string, string>();
- for (int i = 0; i < files.Count; i++)
- if (tmp.ContainsKey(files[i].Name))
- {
- files.RemoveAt(i);
- i--;
- }
- else
- tmp.Add(files[i].Name, null);
-
- return files;
}
public void RenameFile(string bucketName, string source, string target)
diff --git a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs index 503ea4d80..1131c521f 100644 --- a/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs +++ b/Duplicati/Library/Backend/SSHv2/SSHv2Backend.cs @@ -106,7 +106,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -313,10 +313,8 @@ namespace Duplicati.Library.Backend }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- var files = new List<IFileEntry>();
-
string path = ".";
CreateConnection();
@@ -324,9 +322,7 @@ namespace Duplicati.Library.Backend foreach (Renci.SshNet.Sftp.SftpFile ls in m_con.ListDirectory(path))
if (ls.Name.ToString() != "." && ls.Name.ToString() != "..")
- files.Add(new FileEntry(ls.Name.ToString(), ls.Length, ls.LastAccessTime, ls.LastWriteTime) { IsFolder = ls.Attributes.IsDirectory });
-
- return files;
+ yield return new FileEntry(ls.Name.ToString(), ls.Length, ls.LastAccessTime, ls.LastWriteTime) { IsFolder = ls.Attributes.IsDirectory };
}
public static Renci.SshNet.PrivateKeyFile ValidateKeyFile(string filename, string password)
diff --git a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs index 4793cf9a1..8e837e39f 100644 --- a/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs +++ b/Duplicati/Library/Backend/SharePoint/SharePointBackend.cs @@ -393,8 +393,8 @@ namespace Duplicati.Library.Backend testContextForWeb(ctx, true); } - public List<IFileEntry> List() { return doList(false); } - private List<IFileEntry> doList(bool useNewContext) + public IEnumerable<IFileEntry> List() { return doList(false); } + private IEnumerable<IFileEntry> doList(bool useNewContext) { SP.ClientContext ctx = getSpClientContext(useNewContext); try @@ -407,20 +407,7 @@ namespace Duplicati.Library.Backend if (!remoteFolder.Exists) throw new Interface.FolderMissingException(Strings.SharePoint.MissingElementError(m_serverRelPath, m_spWebUrl)); - List<IFileEntry> files = new List<IFileEntry>(remoteFolder.Folders.Count + remoteFolder.Files.Count); - foreach (var f in remoteFolder.Folders.Where(ff => ff.Exists)) - { - FileEntry fe = new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated - fe.IsFolder = true; - files.Add(fe); - } - foreach (var f in remoteFolder.Files.Where(ff => ff.Exists)) - { - FileEntry fe = new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated - fe.IsFolder = false; - files.Add(fe); - } - return files; + return doListWithoutExceptionCatch(remoteFolder); } catch (ServerException) { throw; /* rethrow if Server answered */ } catch (Interface.FileMissingException) { throw; } @@ -429,6 +416,22 @@ namespace Duplicati.Library.Backend finally { } } + private IEnumerable<IFileEntry> doListWithoutExceptionCatch(SP.Folder remoteFolder) + { + foreach (var f in remoteFolder.Folders.Where(ff => ff.Exists)) + { + FileEntry fe = new FileEntry(f.Name, -1, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated + fe.IsFolder = true; + yield return fe; + } + foreach (var f in remoteFolder.Files.Where(ff => ff.Exists)) + { + FileEntry fe = new FileEntry(f.Name, f.Length, f.TimeLastModified, f.TimeLastModified); // f.TimeCreated + fe.IsFolder = false; + yield return fe; + } + } + public void Get(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.Create(filename)) diff --git a/Duplicati/Library/Backend/Sia/Sia.cs b/Duplicati/Library/Backend/Sia/Sia.cs index a0ff731c0..2899d75aa 100644 --- a/Duplicati/Library/Backend/Sia/Sia.cs +++ b/Duplicati/Library/Backend/Sia/Sia.cs @@ -258,7 +258,7 @@ namespace Duplicati.Library.Backend.Sia public void Test() { - List(); + this.TestList(); } public void CreateFolder() @@ -277,15 +277,23 @@ namespace Duplicati.Library.Backend.Sia } - public List<IFileEntry> List() + public IEnumerable<IFileEntry> List() { - var files = new List<IFileEntry>(); try { SiaFileList fl = GetFiles(); - if (fl.Files == null) - return files; + return ListWithoutExceptionCatch(fl); + } + catch (System.Net.WebException wex) + { + throw new Exception("failed to call /renter/files "+wex.Message); + } + } + private IEnumerable<IFileEntry> ListWithoutExceptionCatch(SiaFileList fl) + { + if (fl.Files != null) + { foreach (var f in fl.Files) { // Sia returns a complete file list, but we're only interested in files that are @@ -295,16 +303,10 @@ namespace Duplicati.Library.Backend.Sia FileEntry fe = new FileEntry(f.Siapath.Substring(m_targetpath.Length + 1)); fe.Size = f.Filesize; fe.IsFolder = false; - files.Add(fe); + yield return fe; } } } - catch (System.Net.WebException wex) - { - throw new Exception("failed to call /renter/files "+wex.Message); - } - - return files; } public void Put(string remotename, string filename) diff --git a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs index 5d09b7c7a..2c78d77a9 100644 --- a/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs +++ b/Duplicati/Library/Backend/TahoeLAFS/TahoeBackend.cs @@ -123,7 +123,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
@@ -145,7 +145,7 @@ namespace Duplicati.Library.Backend get { return "tahoe"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
TahoeEl data;
@@ -184,7 +184,6 @@ namespace Duplicati.Library.Backend if (data == null || data.node == null || data.nodetype != "dirnode")
throw new Exception("Invalid folder listing response");
- var files = new List<IFileEntry>();
foreach (var e in data.node.children)
{
if (e.Value == null || e.Value.node == null)
@@ -205,10 +204,8 @@ namespace Duplicati.Library.Backend if (isFile)
fe.Size = e.Value.node.size;
- files.Add(fe);
+ yield return fe;
}
-
- return files;
}
public void Put(string remotename, string filename)
diff --git a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs index 6662755f8..d35815f16 100644 --- a/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs +++ b/Duplicati/Library/Backend/WEBDAV/WEBDAV.cs @@ -125,126 +125,129 @@ namespace Duplicati.Library.Backend get { return "webdav"; }
}
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
try
{
- var req = CreateRequest("");
+ return this.ListWithouExceptionCatch();
+ }
+ catch (System.Net.WebException wex)
+ {
+ if (wex.Response as System.Net.HttpWebResponse != null &&
+ ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict))
+ throw new Interface.FolderMissingException(Strings.WEBDAV.MissingFolderError(m_path, wex.Message), wex);
- req.Method = "PROPFIND";
- req.Headers.Add("Depth", "1");
- req.ContentType = "text/xml";
- req.ContentLength = PROPFIND_BODY.Length;
+ if (wex.Response as System.Net.HttpWebResponse != null && (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.MethodNotAllowed)
+ throw new UserInformationException(Strings.WEBDAV.MethodNotAllowedError((wex.Response as System.Net.HttpWebResponse).StatusCode), wex);
- var areq = new Utility.AsyncHttpRequest(req);
- using (System.IO.Stream s = areq.GetRequestStream())
- s.Write(PROPFIND_BODY, 0, PROPFIND_BODY.Length);
-
- var doc = new System.Xml.XmlDocument();
- using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
- {
- int code = (int)resp.StatusCode;
- if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
- throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
+ throw;
+ }
+ }
- if (!string.IsNullOrEmpty(m_debugPropfindFile))
- {
- using (var rs = areq.GetResponseStream())
- using (var fs = new System.IO.FileStream(m_debugPropfindFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
- Utility.Utility.CopyStream(rs, fs, false, m_copybuffer);
+ private IEnumerable<IFileEntry> ListWithouExceptionCatch()
+ {
+ var req = CreateRequest("");
- doc.Load(m_debugPropfindFile);
- }
- else
- {
- using (var rs = areq.GetResponseStream())
- doc.Load(rs);
- }
- }
+ req.Method = "PROPFIND";
+ req.Headers.Add("Depth", "1");
+ req.ContentType = "text/xml";
+ req.ContentLength = PROPFIND_BODY.Length;
- System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
- nm.AddNamespace("D", "DAV:");
+ var areq = new Utility.AsyncHttpRequest(req);
+ using (System.IO.Stream s = areq.GetRequestStream())
+ s.Write(PROPFIND_BODY, 0, PROPFIND_BODY.Length);
- List<IFileEntry> files = new List<IFileEntry>();
- m_filenamelist = new List<string>();
+ var doc = new System.Xml.XmlDocument();
+ using (var resp = (System.Net.HttpWebResponse)areq.GetResponse())
+ {
+ int code = (int)resp.StatusCode;
+ if (code < 200 || code >= 300) //For some reason Mono does not throw this automatically
+ throw new System.Net.WebException(resp.StatusDescription, null, System.Net.WebExceptionStatus.ProtocolError, resp);
- foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
+ if (!string.IsNullOrEmpty(m_debugPropfindFile))
{
- //IIS uses %20 for spaces and %2B for +
- //Apache uses %20 for spaces and + for +
- string name = Library.Utility.Uri.UrlDecode(n.InnerText.Replace("+", "%2B"));
-
- string cmp_path;
-
- //TODO: This list is getting ridiculous, should change to regexps
-
- if (name.StartsWith(m_url))
- cmp_path = m_url;
- else if (name.StartsWith(m_rawurl))
- cmp_path = m_rawurl;
- else if (name.StartsWith(m_rawurlPort))
- cmp_path = m_rawurlPort;
- else if (name.StartsWith(m_path))
- cmp_path = m_path;
- else if (name.StartsWith("/" + m_path))
- cmp_path = "/" + m_path;
- else if (name.StartsWith(m_sanitizedUrl))
- cmp_path = m_sanitizedUrl;
- else if (name.StartsWith(m_reverseProtocolUrl))
- cmp_path = m_reverseProtocolUrl;
- else
- continue;
-
- if (name.Length <= cmp_path.Length)
- continue;
-
- name = name.Substring(cmp_path.Length);
-
- long size = -1;
- DateTime lastAccess = new DateTime();
- DateTime lastModified = new DateTime();
- bool isCollection = false;
-
- System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
- if (stat != null)
- {
- System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
- if (s != null)
- size = long.Parse(s.InnerText);
- s = stat.SelectSingleNode("D:getlastmodified", nm);
- if (s != null)
- try
- {
- //Not important if this succeeds
- lastAccess = lastModified = DateTime.Parse(s.InnerText, System.Globalization.CultureInfo.InvariantCulture);
- }
- catch { }
-
- s = stat.SelectSingleNode("D:iscollection", nm);
- if (s != null)
- isCollection = s.InnerText.Trim() == "1";
- else
- isCollection = (stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null);
- }
-
- FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
- fe.IsFolder = isCollection;
- files.Add(fe);
- m_filenamelist.Add(name);
- }
+ using (var rs = areq.GetResponseStream())
+ using (var fs = new System.IO.FileStream(m_debugPropfindFile, System.IO.FileMode.Create, System.IO.FileAccess.Write, System.IO.FileShare.None))
+ Utility.Utility.CopyStream(rs, fs, false, m_copybuffer);
- return files;
+ doc.Load(m_debugPropfindFile);
+ }
+ else
+ {
+ using (var rs = areq.GetResponseStream())
+ doc.Load(rs);
+ }
}
- catch (System.Net.WebException wex)
+
+ System.Xml.XmlNamespaceManager nm = new System.Xml.XmlNamespaceManager(doc.NameTable);
+ nm.AddNamespace("D", "DAV:");
+
+ List<IFileEntry> files = new List<IFileEntry>();
+ m_filenamelist = new List<string>();
+
+ foreach (System.Xml.XmlNode n in doc.SelectNodes("D:multistatus/D:response/D:href", nm))
{
- if (wex.Response as System.Net.HttpWebResponse != null &&
- ((wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.NotFound || (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.Conflict))
- throw new Interface.FolderMissingException(Strings.WEBDAV.MissingFolderError(m_path, wex.Message), wex);
+ //IIS uses %20 for spaces and %2B for +
+ //Apache uses %20 for spaces and + for +
+ string name = Library.Utility.Uri.UrlDecode(n.InnerText.Replace("+", "%2B"));
+
+ string cmp_path;
+
+ //TODO: This list is getting ridiculous, should change to regexps
+
+ if (name.StartsWith(m_url))
+ cmp_path = m_url;
+ else if (name.StartsWith(m_rawurl))
+ cmp_path = m_rawurl;
+ else if (name.StartsWith(m_rawurlPort))
+ cmp_path = m_rawurlPort;
+ else if (name.StartsWith(m_path))
+ cmp_path = m_path;
+ else if (name.StartsWith("/" + m_path))
+ cmp_path = "/" + m_path;
+ else if (name.StartsWith(m_sanitizedUrl))
+ cmp_path = m_sanitizedUrl;
+ else if (name.StartsWith(m_reverseProtocolUrl))
+ cmp_path = m_reverseProtocolUrl;
+ else
+ continue;
- if (wex.Response as System.Net.HttpWebResponse != null && (wex.Response as System.Net.HttpWebResponse).StatusCode == System.Net.HttpStatusCode.MethodNotAllowed)
- throw new UserInformationException(Strings.WEBDAV.MethodNotAllowedError((wex.Response as System.Net.HttpWebResponse).StatusCode), wex);
+ if (name.Length <= cmp_path.Length)
+ continue;
+
+ name = name.Substring(cmp_path.Length);
+
+ long size = -1;
+ DateTime lastAccess = new DateTime();
+ DateTime lastModified = new DateTime();
+ bool isCollection = false;
+
+ System.Xml.XmlNode stat = n.ParentNode.SelectSingleNode("D:propstat/D:prop", nm);
+ if (stat != null)
+ {
+ System.Xml.XmlNode s = stat.SelectSingleNode("D:getcontentlength", nm);
+ if (s != null)
+ size = long.Parse(s.InnerText);
+ s = stat.SelectSingleNode("D:getlastmodified", nm);
+ if (s != null)
+ try
+ {
+ //Not important if this succeeds
+ lastAccess = lastModified = DateTime.Parse(s.InnerText, System.Globalization.CultureInfo.InvariantCulture);
+ }
+ catch { }
+
+ s = stat.SelectSingleNode("D:iscollection", nm);
+ if (s != null)
+ isCollection = s.InnerText.Trim() == "1";
+ else
+ isCollection = (stat.SelectSingleNode("D:resourcetype/D:collection", nm) != null);
+ }
- throw;
+ FileEntry fe = new FileEntry(name, size, lastAccess, lastModified);
+ fe.IsFolder = isCollection;
+ m_filenamelist.Add(name);
+ yield return fe;
}
}
@@ -308,7 +311,7 @@ namespace Duplicati.Library.Backend public void Test()
{
- List();
+ this.TestList();
}
public void CreateFolder()
diff --git a/Duplicati/Library/Interface/BackendExtensions.cs b/Duplicati/Library/Interface/BackendExtensions.cs new file mode 100644 index 000000000..486632d76 --- /dev/null +++ b/Duplicati/Library/Interface/BackendExtensions.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Interface +{ + /// <summary> + /// Contains extension methods for the IBackend interfaces + /// </summary> + public static class BackendExtensions + { + /// <summary> + /// Tests a backend by invoking the List() method. + /// As long as the iteration can either complete or find at least one file without throwing, the test is successful + /// </summary> + /// <param name="backend">Backend to test</param> + public static void TestList(this IBackend backend) + { + // If we can iterate successfully, even if it's empty, then the backend test is successful + foreach (IFileEntry file in backend.List()) + { + break; + } + } + } +} diff --git a/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj b/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj index 5a2b82a5c..59d5026e0 100644 --- a/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj +++ b/Duplicati/Library/Interface/Duplicati.Library.Interface.csproj @@ -42,6 +42,7 @@ <Reference Include="System" />
</ItemGroup>
<ItemGroup>
+ <Compile Include="BackendExtensions.cs" />
<Compile Include="CommandLineArgument.cs" />
<Compile Include="CommonStrings.cs" />
<Compile Include="CustomExceptions.cs" />
diff --git a/Duplicati/Library/Interface/IBackend.cs b/Duplicati/Library/Interface/IBackend.cs index a8b5532b9..8281a9372 100644 --- a/Duplicati/Library/Interface/IBackend.cs +++ b/Duplicati/Library/Interface/IBackend.cs @@ -46,10 +46,10 @@ namespace Duplicati.Library.Interface string ProtocolKey { get; }
/// <summary>
- /// Returns a list of files found on the remote location
+ /// Enumerates a list of files found on the remote location
/// </summary>
/// <returns>The list of files</returns>
- List<IFileEntry> List();
+ IEnumerable<IFileEntry> List();
/// <summary>
/// Puts the content of the file to the url passed
diff --git a/Duplicati/Library/Main/BackendManager.cs b/Duplicati/Library/Main/BackendManager.cs index 8e56ac660..bae356016 100644 --- a/Duplicati/Library/Main/BackendManager.cs +++ b/Duplicati/Library/Main/BackendManager.cs @@ -1087,7 +1087,7 @@ namespace Duplicati.Library.Main {
m_statwriter.SendEvent(BackendActionType.List, BackendEventType.Started, null, -1);
- var r = m_backend.List();
+ var r = m_backend.List().ToList();
StringBuilder sb = new StringBuilder();
sb.AppendLine("[");
diff --git a/Duplicati/UnitTest/RandomErrorBackend.cs b/Duplicati/UnitTest/RandomErrorBackend.cs index 6f14115c5..4151a8622 100644 --- a/Duplicati/UnitTest/RandomErrorBackend.cs +++ b/Duplicati/UnitTest/RandomErrorBackend.cs @@ -23,6 +23,6 @@ namespace Duplicati.UnitTest public RandomErrorBackend()
{
}
public RandomErrorBackend(string url, Dictionary<string, string> options)
{
var u = new Library.Utility.Uri(url).SetScheme(WrappedBackend).ToString();
m_backend = (IStreamingBackend)Library.DynamicLoader.BackendLoader.GetBackend(u, options);
}
private void ThrowErrorRandom()
{
if (random.NextDouble() > 0.90)
throw new Exception("Random upload failure");
}
- #region IStreamingBackend implementation
public void Put(string remotename, Stream stream)
{
var uploadError = random.NextDouble() > 0.9;
using(var f = new Library.Utility.ProgressReportingStream(stream, stream.Length, x => { if (uploadError && stream.Position > stream.Length / 2) throw new Exception("Random upload failure"); }))
m_backend.Put(remotename, f);
ThrowErrorRandom();
}
public void Get(string remotename, Stream stream)
{
ThrowErrorRandom();
m_backend.Get(remotename, stream);
ThrowErrorRandom();
}
#endregion
#region IBackend implementation
public List<IFileEntry> List()
{
return m_backend.List();
}
public void Put(string remotename, string filename)
{
ThrowErrorRandom();
m_backend.Put(remotename, filename);
ThrowErrorRandom();
}
public void Get(string remotename, string filename)
{
ThrowErrorRandom();
m_backend.Get(remotename, filename);
ThrowErrorRandom();
}
public void Delete(string remotename)
{
ThrowErrorRandom();
m_backend.Delete(remotename);
ThrowErrorRandom();
}
public void Test()
{
m_backend.Test();
}
public void CreateFolder()
{
m_backend.CreateFolder();
}
public string DisplayName
{
get
{
return "Random Error Backend";
}
}
public string ProtocolKey
{
get
{
return "randomerror";
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
if (m_backend == null)
try { return Duplicati.Library.DynamicLoader.BackendLoader.GetSupportedCommands(WrappedBackend + "://"); }
catch { }
return m_backend.SupportedCommands;
}
}
public string Description
{
get
{
return "A testing backend that randomly fails";
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_backend != null)
try { m_backend.Dispose(); }
finally { m_backend = null; }
}
#endregion
}
+ #region IStreamingBackend implementation
public void Put(string remotename, Stream stream)
{
var uploadError = random.NextDouble() > 0.9;
using(var f = new Library.Utility.ProgressReportingStream(stream, stream.Length, x => { if (uploadError && stream.Position > stream.Length / 2) throw new Exception("Random upload failure"); }))
m_backend.Put(remotename, f);
ThrowErrorRandom();
}
public void Get(string remotename, Stream stream)
{
ThrowErrorRandom();
m_backend.Get(remotename, stream);
ThrowErrorRandom();
}
#endregion
#region IBackend implementation
public IEnumerable<IFileEntry> List()
{
return m_backend.List();
}
public void Put(string remotename, string filename)
{
ThrowErrorRandom();
m_backend.Put(remotename, filename);
ThrowErrorRandom();
}
public void Get(string remotename, string filename)
{
ThrowErrorRandom();
m_backend.Get(remotename, filename);
ThrowErrorRandom();
}
public void Delete(string remotename)
{
ThrowErrorRandom();
m_backend.Delete(remotename);
ThrowErrorRandom();
}
public void Test()
{
m_backend.Test();
}
public void CreateFolder()
{
m_backend.CreateFolder();
}
public string DisplayName
{
get
{
return "Random Error Backend";
}
}
public string ProtocolKey
{
get
{
return "randomerror";
}
}
public IList<ICommandLineArgument> SupportedCommands
{
get
{
if (m_backend == null)
try { return Duplicati.Library.DynamicLoader.BackendLoader.GetSupportedCommands(WrappedBackend + "://"); }
catch { }
return m_backend.SupportedCommands;
}
}
public string Description
{
get
{
return "A testing backend that randomly fails";
}
}
#endregion
#region IDisposable implementation
public void Dispose()
{
if (m_backend != null)
try { m_backend.Dispose(); }
finally { m_backend = null; }
}
#endregion
}
}
diff --git a/Duplicati/UnitTest/SizeOmittingBackend.cs b/Duplicati/UnitTest/SizeOmittingBackend.cs index 57541f28c..baba42555 100644 --- a/Duplicati/UnitTest/SizeOmittingBackend.cs +++ b/Duplicati/UnitTest/SizeOmittingBackend.cs @@ -51,13 +51,12 @@ namespace Duplicati.UnitTest #endregion
#region IBackend implementation
- public List<IFileEntry> List()
+ public IEnumerable<IFileEntry> List()
{
- return (
+ return
from n in m_backend.List()
where !n.IsFolder
- select new Library.Interface.FileEntry(n.Name)
- ).Cast<Library.Interface.IFileEntry>().ToList();
+ select new Library.Interface.FileEntry(n.Name);
}
public void Put(string remotename, string filename)
{
|