From 3600eed64d8a0b512375e984196707acd5a416a2 Mon Sep 17 00:00:00 2001 From: TopperDEL Date: Tue, 19 May 2020 14:01:44 +0200 Subject: fix: Move the tardigrad-library to the correct location --- .../Duplicati.Library.Backend.Tardigrade.csproj | 85 ++++++ .../Backend/Tardigrade/Properties/AssemblyInfo.cs | 36 +++ Duplicati/Library/Backend/Tardigrade/Strings.cs | 30 ++ .../Backend/Tardigrade/TardigradeBackend.cs | 330 +++++++++++++++++++++ .../Library/Backend/Tardigrade/TardigradeConfig.cs | 68 +++++ .../Library/Backend/Tardigrade/TardigradeFile.cs | 56 ++++ .../Library/Backend/Tardigrade/packages.config | 4 + 7 files changed, 609 insertions(+) create mode 100644 Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj create mode 100644 Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs create mode 100644 Duplicati/Library/Backend/Tardigrade/Strings.cs create mode 100644 Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs create mode 100644 Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs create mode 100644 Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs create mode 100644 Duplicati/Library/Backend/Tardigrade/packages.config (limited to 'Duplicati/Library/Backend/Tardigrade') diff --git a/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj b/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj new file mode 100644 index 000000000..cb6eda2b3 --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj @@ -0,0 +1,85 @@ + + + + + Debug + AnyCPU + {AE035E01-C917-4F13-A35E-78F21C1A2F17} + Library + Properties + Duplicati.Library.Backend.Tardigrade + Duplicati.Library.Backend.Tardigrade + v4.6.2 + 512 + true + + + + + true + full + false + bin\Debug\ + DEBUG;TRACE + prompt + 4 + x64 + + + pdbonly + true + bin\Release\ + TRACE + prompt + 4 + + + + + + + + + + + + ..\packages\uplink.NET.2.1.5\lib\netstandard2.0\uplink.NET.dll + + + + + + + + + + + + {d63e53e4-a458-4c2f-914d-92f715f58acf} + Duplicati.Library.Common + + + {c5899f45-b0ff-483c-9d38-24a9fcaab237} + Duplicati.Library.Interface + + + {b68f2214-951f-4f78-8488-66e1ed3f50bf} + Duplicati.Library.Localization + + + {de3e5d4c-51ab-4e5e-bee8-e636cebfba65} + Duplicati.Library.Utility + + + + + + + + + + Dieses Projekt verweist auf mindestens ein NuGet-Paket, das auf diesem Computer fehlt. Verwenden Sie die Wiederherstellung von NuGet-Paketen, um die fehlenden Dateien herunterzuladen. Weitere Informationen finden Sie unter "http://go.microsoft.com/fwlink/?LinkID=322105". Die fehlende Datei ist "{0}". + + + + \ No newline at end of file diff --git a/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs b/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..5f3cc907e --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs @@ -0,0 +1,36 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// Allgemeine Informationen über eine Assembly werden über die folgenden +// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern, +// die einer Assembly zugeordnet sind. +[assembly: AssemblyTitle("Duplicati.Library.Backend.Tardigrade")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Duplicati.Library.Backend.Tardigrade")] +[assembly: AssemblyCopyright("Copyright © 2020")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Durch Festlegen von ComVisible auf FALSE werden die Typen in dieser Assembly +// für COM-Komponenten unsichtbar. Wenn Sie auf einen Typ in dieser Assembly von +// COM aus zugreifen müssen, sollten Sie das ComVisible-Attribut für diesen Typ auf "True" festlegen. +[assembly: ComVisible(false)] + +// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird +[assembly: Guid("ae035e01-c917-4f13-a35e-78f21c1a2f17")] + +// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten: +// +// Hauptversion +// Nebenversion +// Buildnummer +// Revision +// +// Sie können alle Werte angeben oder Standardwerte für die Build- und Revisionsnummern verwenden, +// indem Sie "*" wie unten gezeigt eingeben: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] diff --git a/Duplicati/Library/Backend/Tardigrade/Strings.cs b/Duplicati/Library/Backend/Tardigrade/Strings.cs new file mode 100644 index 000000000..7540c2859 --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/Strings.cs @@ -0,0 +1,30 @@ +using Duplicati.Library.Localization.Short; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Backend.Strings +{ + internal static class Tardigrade + { + public static string DisplayName { get { return LC.L(@"Tardigrade Decentralized Cloud Storage"); } } + public static string Description { get { return LC.L(@"This backend can read and write data to the Tardigrade Decentralized Cloud Storage."); } } + public static string TestConnectionFailed { get { return LC.L(@"The connection-test failed."); } } + public static string TardigradeAuthMethodDescriptionShort { get { return LC.L(@"The authentication method"); } } + public static string TardigradeAuthMethodDescriptionLong { get { return LC.L(@"The authentication method describes which way to use to connect to the network - either via API key or via an access grant."); } } + public static string TardigradeSatelliteDescriptionShort { get { return LC.L(@"The satellite"); } } + public static string TardigradeSatelliteDescriptionLong { get { return LC.L(@"The satellite that keeps track of all metadata. Use a Tardigrade-grade server for high-performance SLA-backed connectivity or use a community server. Or even host your own."); } } + public static string TardigradeAPIKeyDescriptionShort { get { return LC.L(@"The API key"); } } + public static string TardigradeAPIKeyDescriptionLong { get { return LC.L(@"The API key grants access to a specific project on your chosen satellite. Head over to the dashboard of your satellite to create one if you do not already have an API key."); } } + public static string TardigradeSecretDescriptionShort { get { return LC.L(@"The encryption passphrase"); } } + public static string TardigradeSecretDescriptionLong { get { return LC.L(@"The encryption passphrase is used to encrypt your data before sending it to the tardigrade network. This passphrase can be the only secret to provide - for Tardigrade you do not necessary need any additional encryption (from Duplicati) in place."); } } + public static string TardigradeSharedAccessDescriptionShort { get { return LC.L(@"The access grant"); } } + public static string TardigradeSharedAccessDescriptionLong { get { return LC.L(@"An access grant contains all information in one encrypted string. You may use it instead of a satellite, API key and secret."); } } + public static string TardigradeBucketDescriptionShort { get { return LC.L(@"The bucket"); } } + public static string TardigradeBucketDescriptionLong { get { return LC.L(@"The bucket where the backup will reside in."); } } + public static string TardigradeFolderDescriptionShort { get { return LC.L(@"The folder"); } } + public static string TardigradeFolderDescriptionLong { get { return LC.L(@"The folder within the bucket where the backup will reside in."); } } + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs b/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs new file mode 100644 index 000000000..58a1f7343 --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs @@ -0,0 +1,330 @@ +using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using uplink.NET.Interfaces; +using uplink.NET.Models; +using uplink.NET.Services; + +namespace Duplicati.Library.Backend.Tardigrade +{ + public class Tardigrade : IStreamingBackend + { + private const string TARDIGRADE_AUTH_METHOD = "tardigrade-auth-method"; + private const string TARDIGRADE_SATELLITE = "tardigrade-satellite"; + private const string TARDIGRADE_API_KEY = "tardigrade-api-key"; + private const string TARDIGRADE_SECRET = "tardigrade-secret"; + private const string TARDIGRADE_SHARED_ACCESS = "tardigrade-shared-access"; + private const string TARDIGRADE_BUCKET = "tardigrade-bucket"; + private const string TARDIGRADE_FOLDER = "tardigrade-folder"; + private const string PROTOCOL_KEY = "tardigrade"; + + private readonly string _satellite; + private readonly string _api_key; + private readonly string _secret; + private readonly string _bucket; + private readonly string _folder; + private Access _access; + private IBucketService _bucketService; + private IObjectService _objectService; + + public static readonly Dictionary KNOWN_TARDIGRADE_SATELLITES = new Dictionary(StringComparer.OrdinalIgnoreCase){ + { "US Central 1", "us-central-1.tardigrade.io:7777" }, + { "Asia East 1", "asia-east-1.tardigrade.io:7777" }, + { "Europe West 1", "europe-west-1.tardigrade.io:7777" }, + }; + + public static readonly Dictionary KNOWN_AUTHENTICATION_METHODS = new Dictionary(StringComparer.OrdinalIgnoreCase){ + { "API key", "API key" }, + { "Access grant", "Access grant" }, + }; + + // ReSharper disable once UnusedMember.Global + // This constructor is needed by the BackendLoader. + public Tardigrade() + { + } + + // ReSharper disable once UnusedMember.Global + // This constructor is needed by the BackendLoader. + public Tardigrade(string url, Dictionary options) + { + var auth_method = options[TARDIGRADE_AUTH_METHOD]; + + Access.SetTempDirectory(System.IO.Path.GetTempPath()); + + if (auth_method == "Access grant") + { + //Create an access from the access grant + var shared_access = options[TARDIGRADE_SHARED_ACCESS]; + _access = new Access(shared_access); + } + else + { + //Create an access for a satellite, API key and encryption passphrase + _satellite = options[TARDIGRADE_SATELLITE]; + + if (options.ContainsKey(TARDIGRADE_API_KEY)) + { + _api_key = options[TARDIGRADE_API_KEY]; + } + if (options.ContainsKey(TARDIGRADE_SECRET)) + { + _secret = options[TARDIGRADE_SECRET]; + } + + _access = new Access(_satellite, _api_key, _secret); + } + + _bucketService = new BucketService(_access); + _objectService = new ObjectService(_access); + + //If no bucket was provided use the default "duplicati"-bucket + if (options.ContainsKey(TARDIGRADE_BUCKET)) + { + _bucket = options[TARDIGRADE_BUCKET]; + } + else + { + _bucket = "duplicati"; + } + + if (options.ContainsKey(TARDIGRADE_FOLDER)) + { + _folder = options[TARDIGRADE_FOLDER]; + } + } + + public string DisplayName + { + get { return Strings.Tardigrade.DisplayName; } + } + + public string ProtocolKey => PROTOCOL_KEY; + + public IList SupportedCommands + { + get + { + return new List(new ICommandLineArgument[] { + new CommandLineArgument(TARDIGRADE_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeAuthMethodDescriptionShort, Strings.Tardigrade.TardigradeAuthMethodDescriptionLong, "API key"), + new CommandLineArgument(TARDIGRADE_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeSatelliteDescriptionShort, Strings.Tardigrade.TardigradeSatelliteDescriptionLong, "us-central-1.tardigrade.io:7777"), + new CommandLineArgument(TARDIGRADE_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeAPIKeyDescriptionShort, Strings.Tardigrade.TardigradeAPIKeyDescriptionLong), + new CommandLineArgument(TARDIGRADE_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Tardigrade.TardigradeSecretDescriptionShort, Strings.Tardigrade.TardigradeSecretDescriptionLong), + new CommandLineArgument(TARDIGRADE_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeSharedAccessDescriptionShort, Strings.Tardigrade.TardigradeSharedAccessDescriptionLong), + new CommandLineArgument(TARDIGRADE_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeBucketDescriptionShort, Strings.Tardigrade.TardigradeBucketDescriptionLong), + new CommandLineArgument(TARDIGRADE_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeFolderDescriptionShort, Strings.Tardigrade.TardigradeFolderDescriptionLong), + }); + } + } + + public string Description + { + get + { + return Strings.Tardigrade.Description; + } + } + + public string[] DNSName + { + get + { + return new string[0]; + } + } + + public void CreateFolder() + { + //Tardigrade has no folders + } + + public void Delete(string remotename) + { + var deleteTask = DeleteAsync(remotename); + deleteTask.Wait(); + } + + public async Task DeleteAsync(string remotename) + { + try + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename); + } + catch (Exception root) + { + throw new FileMissingException(root); + } + } + + public void Dispose() + { + if (_objectService != null) + { + _objectService = null; + } + if (_bucketService != null) + { + _bucketService = null; + } + if (_access != null) + { + _access.Dispose(); + _access = null; + } + } + + public void Get(string remotename, string filename) + { + var getTask = GetAsync(remotename, filename); + getTask.Wait(); + } + + public async Task GetAsync(string remotename, string filename) + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); + await download.StartDownloadAsync(); + + if (download.Completed) + { + using (FileStream file = new FileStream(filename, FileMode.Create)) + { + await file.WriteAsync(download.DownloadedBytes, 0, (int)download.BytesReceived); + await file.FlushAsync().ConfigureAwait(false); + } + } + } + + public void Get(string remotename, Stream stream) + { + var getTask = GetAsync(remotename, stream); + getTask.Wait(); + } + + public async Task GetAsync(string remotename, Stream stream) + { + int index = 0; + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); + download.DownloadOperationProgressChanged += (op) => + { + int newPartLength = (int)op.BytesReceived - index; + byte[] newPart = new byte[newPartLength]; + Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength); + stream.Write(newPart, 0, newPartLength); + index = index + newPartLength; + }; + await download.StartDownloadAsync(); + } + + public IEnumerable List() + { + var listTask = ListAsync(); + listTask.Wait(); + return listTask.Result; + } + + private async Task> ListAsync() + { + List files = new List(); + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true }); + + foreach (var obj in objects.Items) + { + TardigradeFile file = new TardigradeFile(obj); + file.Name = file.Name.Replace(GetBasePath(), ""); + files.Add(file); + } + + return files; + } + + public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) + { + using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) + await PutAsync(remotename, fs, cancelToken); + } + + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + CustomMetadata custom = new CustomMetadata(); + custom.Entries.Add(new CustomMetadataEntry { Key = TardigradeFile.TARDIGRADE_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") }); + custom.Entries.Add(new CustomMetadataEntry { Key = TardigradeFile.TARDIGRADE_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") }); + var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false); + await upload.StartUploadAsync(); + } + + public void Test() + { + var testTask = TestAsync(); + testTask.Wait(10000); + if (!testTask.Result) + { + throw new Exception(Strings.Tardigrade.TestConnectionFailed); + } + } + + /// + /// Test the connection by: + /// - creating the bucket (if it not already exists) + /// - uploading 256 random bytes to a test-file + /// - downloading the file back and expecting 256 bytes + /// + /// true, if the test was successfull or and exception + private async Task TestAsync() + { + string testFileName = GetBasePath() + "duplicati_test.dat"; + + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false); + await upload.StartUploadAsync(); + + var download = await _objectService.DownloadObjectAsync(bucket, testFileName, new DownloadOptions(), false); + await download.StartDownloadAsync(); + + await _objectService.DeleteObjectAsync(bucket, testFileName); + + if (download.Failed || download.BytesReceived != 256) + { + throw new Exception(download.ErrorMessage); + } + + return true; + } + + /// + /// Gets the base path - depending on there is a folder set or not + /// + /// The base path within a bucket where the backup shall be placed + private string GetBasePath() + { + if (!string.IsNullOrEmpty(_folder)) + return _folder + "/"; + else + return ""; + } + + /// + /// Creates some random bytes with the given length - just for testing the connection + /// + /// The length of the bytes to create + /// A byte-array with the given length + private static byte[] GetRandomBytes(long length) + { + byte[] bytes = new byte[length]; + Random rand = new Random(); + rand.NextBytes(bytes); + + return bytes; + } + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs b/Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs new file mode 100644 index 000000000..1464fe17f --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs @@ -0,0 +1,68 @@ +using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Backend.Tardigrade +{ + public class TardigradeConfig : IWebModule + { + private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Satellites; + private const string KEY_CONFIGTYPE = "tardigrade-config"; + private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); + + public enum ConfigType + { + Satellites, + AuthenticationMethods + } + + #region IWebModule implementation + + public string Key { get { return "tardigrade-getconfig"; } } + + public string DisplayName { get { return "Tardigrade configuration module"; } } + + public string Description { get { return "Exposes Tardigrade configuration as a web module"; } } + + public IList SupportedCommands + { + get + { + return new List(new ICommandLineArgument[] { + new CommandLineArgument(KEY_CONFIGTYPE, CommandLineArgument.ArgumentType.Enumeration, "The config to get", "Provides different config values", DEFAULT_CONFIG_TYPE_STR, Enum.GetNames(typeof(ConfigType))) + + }); + } + } + + public IDictionary Execute(IDictionary options) + { + string k; + options.TryGetValue(KEY_CONFIGTYPE, out k); + if (string.IsNullOrWhiteSpace(k)) + { + k = DEFAULT_CONFIG_TYPE_STR; + } + + ConfigType ct; + if (!Enum.TryParse(k, true, out ct)) + { + ct = DEFAULT_CONFIG_TYPE; + } + + switch (ct) + { + case ConfigType.Satellites: + return Tardigrade.KNOWN_TARDIGRADE_SATELLITES; + case ConfigType.AuthenticationMethods: + return Tardigrade.KNOWN_AUTHENTICATION_METHODS; + default: + return Tardigrade.KNOWN_TARDIGRADE_SATELLITES; + } + } + #endregion + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs b/Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs new file mode 100644 index 000000000..794e427e0 --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs @@ -0,0 +1,56 @@ +using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Backend.Tardigrade +{ + public class TardigradeFile : IFileEntry + { + public static readonly string TARDIGRADE_LAST_ACCESS = "DUPLICATI:LAST-ACCESS"; + public static readonly string TARDIGRADE_LAST_MODIFICATION = "DUPLICATI:LAST-MODIFICATION"; + public bool IsFolder { get; set; } + + public DateTime LastAccess { get; set; } + + public DateTime LastModification { get; set; } + + public string Name { get; set; } + + public long Size { get; set; } + + public TardigradeFile() + { + + } + + public TardigradeFile(uplink.NET.Models.Object tardigradeObject) + { + IsFolder = tardigradeObject.IsPrefix; + var lastAccess = tardigradeObject.CustomMetaData.Entries.Where(e => e.Key == TARDIGRADE_LAST_ACCESS).FirstOrDefault(); + if (lastAccess != null && !string.IsNullOrEmpty(lastAccess.Value)) + { + LastAccess = DateTime.Parse(lastAccess.Value); + } + else + { + LastAccess = DateTime.MinValue; + } + + var lastMod = tardigradeObject.CustomMetaData.Entries.Where(e => e.Key == TARDIGRADE_LAST_MODIFICATION).FirstOrDefault(); + if (lastMod != null && !string.IsNullOrEmpty(lastMod.Value)) + { + LastModification = DateTime.Parse(lastMod.Value); + } + else + { + LastModification = DateTime.MinValue; + } + + Name = tardigradeObject.Key; + Size = tardigradeObject.SystemMetaData.ContentLength; + } + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/packages.config b/Duplicati/Library/Backend/Tardigrade/packages.config new file mode 100644 index 000000000..3e0da8d53 --- /dev/null +++ b/Duplicati/Library/Backend/Tardigrade/packages.config @@ -0,0 +1,4 @@ + + + + \ No newline at end of file -- cgit v1.2.3