diff options
| author | warwickmm <warwickmm@users.noreply.github.com> | 2021-07-20 18:28:54 +0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2021-07-20 18:28:54 +0300 |
| commit | 184dfeae7860cd34019baf3d3ca635bf6b79dfbe (patch) | |
| tree | 1ec8f50736a931a3ba46bb108bf6518cad944a05 /Duplicati | |
| parent | a93aa6c9e89a2340dbbc0881657e7abfc5cdabed (diff) | |
| parent | ad63833a32bcd71f62c7136040b073835f353a56 (diff) | |
Merge pull request #4492 from TopperDEL/storj-dcs-update
Add Storj DCS backend and deprecated Tardigrade DCS backend.
Storj renamed their "Tardigrade Decentralized Cloud Storage" product to "Storj DCS". This adds a new Storj DCS backend and deprecates the old Tardigrade one. A warning is issued to inform Tardigrade users to modify their configurations to use the new backend.
This also adds a passphrase verification field to ensure that the user has provided the correct encryption passphrase.
Diffstat (limited to 'Duplicati')
27 files changed, 906 insertions, 387 deletions
diff --git a/Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj b/Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj index 721163ebb..69d035db8 100644 --- a/Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj +++ b/Duplicati/CommandLine/BackendTester/Duplicati.CommandLine.BackendTester.csproj @@ -86,8 +86,12 @@ <Project>{32a74526-3e5f-413a-8cb4-1efdad4c8b91}</Project>
<Name>Duplicati.Library.Backend.Sia</Name>
</ProjectReference>
- <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{ae035e01-c917-4f13-a35e-78f21c1a2f17}</Project>
+ <Name>Duplicati.Library.Backend.Storj</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
<Name>Duplicati.Library.Backend.Tardigrade</Name>
</ProjectReference>
<ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
diff --git a/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj b/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj index 694439e22..f344da03f 100644 --- a/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj +++ b/Duplicati/CommandLine/BackendTool/Duplicati.CommandLine.BackendTool.csproj @@ -92,8 +92,12 @@ <Project>{C0270709-2A40-43B5-8CF1-69581B9FA2A1}</Project>
<Name>Duplicati.Library.Backend.TahoeLAFS</Name>
</ProjectReference>
- <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{ae035e01-c917-4f13-a35e-78f21c1a2f17}</Project>
+ <Name>Duplicati.Library.Backend.Storj</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
<Name>Duplicati.Library.Backend.Tardigrade</Name>
</ProjectReference>
<ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
diff --git a/Duplicati/CommandLine/Duplicati.CommandLine.csproj b/Duplicati/CommandLine/Duplicati.CommandLine.csproj index 83b3926dc..e6fc646ef 100644 --- a/Duplicati/CommandLine/Duplicati.CommandLine.csproj +++ b/Duplicati/CommandLine/Duplicati.CommandLine.csproj @@ -111,6 +111,10 @@ <Project>{32a74526-3e5f-413a-8cb4-1efdad4c8b91}</Project>
<Name>Duplicati.Library.Backend.Sia</Name>
</ProjectReference>
+ <ProjectReference Include="..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
+ <Name>Duplicati.Library.Backend.Tardigrade</Name>
+ </ProjectReference>
<ProjectReference Include="..\Library\Compression\Duplicati.Library.Compression.csproj">
<Project>{19ECCE09-B5EB-406C-8C57-BAC66997D469}</Project>
<Name>Duplicati.Library.Compression</Name>
@@ -231,9 +235,9 @@ <Project>{D63E53E4-A458-4C2F-914D-92F715F58ACF}</Project>
<Name>Duplicati.Library.Common</Name>
</ProjectReference>
- <ProjectReference Include="..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{AE035E01-C917-4F13-A35E-78F21C1A2F17}</Project>
- <Name>Duplicati.Library.Backend.Tardigrade</Name>
+ <Name>Duplicati.Library.Backend.Storj</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
@@ -256,4 +260,4 @@ <Target Name="AfterBuild">
</Target>
-->
-</Project>
+</Project>
\ No newline at end of file diff --git a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj index 819aff870..a4d631b04 100644 --- a/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj +++ b/Duplicati/CommandLine/RecoveryTool/Duplicati.CommandLine.RecoveryTool.csproj @@ -134,6 +134,10 @@ <Project>{C0270709-2A40-43B5-8CF1-69581B9FA2A1}</Project>
<Name>Duplicati.Library.Backend.TahoeLAFS</Name>
</ProjectReference>
+ <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
+ <Name>Duplicati.Library.Backend.Tardigrade</Name>
+ </ProjectReference>
<ProjectReference Include="..\..\Library\Backend\WEBDAV\Duplicati.Library.Backend.WEBDAV.csproj">
<Project>{BAE27510-8B5D-44B2-B33E-372A98908041}</Project>
<Name>Duplicati.Library.Backend.WEBDAV</Name>
@@ -182,9 +186,9 @@ <Project>{D63E53E4-A458-4C2F-914D-92F715F58ACF}</Project>
<Name>Duplicati.Library.Common</Name>
</ProjectReference>
- <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{AE035E01-C917-4F13-A35E-78F21C1A2F17}</Project>
- <Name>Duplicati.Library.Backend.Tardigrade</Name>
+ <Name>Duplicati.Library.Backend.Storj</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
diff --git a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj index e5b3f7b72..df66bc6bc 100644 --- a/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj +++ b/Duplicati/GUI/Duplicati.GUI.TrayIcon/Duplicati.GUI.TrayIcon.csproj @@ -217,8 +217,12 @@ <Project>{C0270709-2A40-43B5-8CF1-69581B9FA2A1}</Project>
<Name>Duplicati.Library.Backend.TahoeLAFS</Name>
</ProjectReference>
- <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{ae035e01-c917-4f13-a35e-78f21c1a2f17}</Project>
+ <Name>Duplicati.Library.Backend.Storj</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
<Name>Duplicati.Library.Backend.Tardigrade</Name>
</ProjectReference>
<ProjectReference Include="..\..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
diff --git a/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj new file mode 100644 index 000000000..4f6179435 --- /dev/null +++ b/Duplicati/Library/Backend/Storj/Duplicati.Library.Backend.Storj.csproj @@ -0,0 +1,128 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="15.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" /> + <PropertyGroup> + <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> + <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> + <ProjectGuid>{AE035E01-C917-4F13-A35E-78F21C1A2F17}</ProjectGuid> + <OutputType>Library</OutputType> + <AppDesignerFolder>Properties</AppDesignerFolder> + <RootNamespace>Duplicati.Library.Backend.Storj</RootNamespace> + <AssemblyName>Duplicati.Library.Backend.Storj</AssemblyName> + <TargetFrameworkVersion>v4.7.1</TargetFrameworkVersion> + <FileAlignment>512</FileAlignment> + <Deterministic>true</Deterministic> + <NuGetPackageImportStamp> + </NuGetPackageImportStamp> + <TargetFrameworkProfile /> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' "> + <DebugSymbols>true</DebugSymbols> + <DebugType>full</DebugType> + <Optimize>false</Optimize> + <OutputPath>bin\Debug\</OutputPath> + <DefineConstants>DEBUG;TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + <PlatformTarget>x64</PlatformTarget> + </PropertyGroup> + <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' "> + <DebugType>pdbonly</DebugType> + <Optimize>true</Optimize> + <OutputPath>bin\Release\</OutputPath> + <DefineConstants>TRACE</DefineConstants> + <ErrorReport>prompt</ErrorReport> + <WarningLevel>4</WarningLevel> + </PropertyGroup> + <ItemGroup> + <Reference Include="SQLite-net, Version=1.7.335.0, Culture=neutral, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\sqlite-net-pcl.1.7.335\lib\netstandard2.0\SQLite-net.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.batteries_v2, Version=2.0.3.851, Culture=neutral, PublicKeyToken=8226ea5df37bcae9, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.bundle_green.2.0.3\lib\net461\SQLitePCLRaw.batteries_v2.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.core, Version=2.0.3.851, Culture=neutral, PublicKeyToken=1488e028ca7ab535, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.core.2.0.3\lib\netstandard2.0\SQLitePCLRaw.core.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.nativelibrary, Version=2.0.3.851, Culture=neutral, PublicKeyToken=502ed628492ab262, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.bundle_green.2.0.3\lib\net461\SQLitePCLRaw.nativelibrary.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.provider.dynamic_cdecl, Version=2.0.3.851, Culture=neutral, PublicKeyToken=b68184102cba0b3b, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.provider.dynamic_cdecl.2.0.3\lib\netstandard2.0\SQLitePCLRaw.provider.dynamic_cdecl.dll</HintPath> + </Reference> + <Reference Include="System" /> + <Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath> + </Reference> + <Reference Include="System.Core" /> + <Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath> + </Reference> + <Reference Include="System.Numerics" /> + <Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath> + </Reference> + <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath> + </Reference> + <Reference Include="System.Xml.Linq" /> + <Reference Include="System.Data.DataSetExtensions" /> + <Reference Include="Microsoft.CSharp" /> + <Reference Include="System.Data" /> + <Reference Include="System.Net.Http" /> + <Reference Include="System.Xml" /> + <Reference Include="uplink.NET, Version=2.7.0.0, Culture=neutral, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\uplink.NET.2.7.1604\lib\netstandard2.0\uplink.NET.dll</HintPath> + </Reference> + </ItemGroup> + <ItemGroup> + <Compile Include="StorjFile.cs" /> + <Compile Include="Strings.cs" /> + <Compile Include="StorjBackend.cs" /> + <Compile Include="Properties\AssemblyInfo.cs" /> + <Compile Include="StorjConfig.cs" /> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="..\..\Common\Duplicati.Library.Common.csproj"> + <Project>{d63e53e4-a458-4c2f-914d-92f715f58acf}</Project> + <Name>Duplicati.Library.Common</Name> + </ProjectReference> + <ProjectReference Include="..\..\Interface\Duplicati.Library.Interface.csproj"> + <Project>{c5899f45-b0ff-483c-9d38-24a9fcaab237}</Project> + <Name>Duplicati.Library.Interface</Name> + </ProjectReference> + <ProjectReference Include="..\..\Localization\Duplicati.Library.Localization.csproj"> + <Project>{b68f2214-951f-4f78-8488-66e1ed3f50bf}</Project> + <Name>Duplicati.Library.Localization</Name> + </ProjectReference> + <ProjectReference Include="..\..\Utility\Duplicati.Library.Utility.csproj"> + <Project>{de3e5d4c-51ab-4e5e-bee8-e636cebfba65}</Project> + <Name>Duplicati.Library.Utility</Name> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <None Include="libstorj_uplink.dylib"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + <None Include="libstorj_uplink.so"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </None> + <None Include="packages.config" /> + </ItemGroup> + <ItemGroup> + <Content Include="win-x64\storj_uplink.dll"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </Content> + <Content Include="win-x86\storj_uplink.dll"> + <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> + </Content> + </ItemGroup> + <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> + <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> + <PropertyGroup> + <ErrorText>Dieses Projekt verweist auf mindestens ein NuGet-Paket, das auf diesem Computer fehlt. Verwenden Sie die Wiederherstellung von NuGet-Paketen, um die fehlenden Dateien herunterzuladen. Weitere Informationen finden Sie unter "http://go.microsoft.com/fwlink/?LinkID=322105". Die fehlende Datei ist "{0}".</ErrorText> + </PropertyGroup> + <Error Condition="!Exists('..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets'))" /> + </Target> + <Import Project="..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets" Condition="Exists('..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets')" /> +</Project>
\ No newline at end of file diff --git a/Duplicati/Library/Backend/Storj/Properties/AssemblyInfo.cs b/Duplicati/Library/Backend/Storj/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..27b1092b7 --- /dev/null +++ b/Duplicati/Library/Backend/Storj/Properties/AssemblyInfo.cs @@ -0,0 +1,37 @@ +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +// Allgemeine Informationen über eine Assembly werden über die folgenden +// Attribute gesteuert. Ändern Sie diese Attributwerte, um die Informationen zu ändern, +// die einer Assembly zugeordnet sind. +[assembly: AssemblyTitle("Duplicati.Library.Backend.Tardigrade")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("Duplicati.Library.Backend.Tardigrade")] +[assembly: AssemblyCopyright("Copyright © 2020")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Durch Festlegen von ComVisible auf FALSE werden die Typen in dieser Assembly +// für COM-Komponenten unsichtbar. Wenn Sie auf einen Typ in dieser Assembly von +// COM aus zugreifen müssen, sollten Sie das ComVisible-Attribut für diesen Typ auf "True" festlegen. +[assembly: ComVisible(false)] + +// Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird +[assembly: Guid("ae035e01-c917-4f13-a35e-78f21c1a2f17")] + +// Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten: +// +// Hauptversion +// Nebenversion +// Buildnummer +// Revision +// +// Sie können alle Werte angeben oder Standardwerte für die Build- und Revisionsnummern verwenden, +// indem Sie "*" wie unten gezeigt eingeben: +// [assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("1.0.0.0")] +[assembly: AssemblyFileVersion("1.0.0.0")] +[assembly:InternalsVisibleTo("Duplicati.Library.Backend.Tardigrade")] diff --git a/Duplicati/Library/Backend/Storj/StorjBackend.cs b/Duplicati/Library/Backend/Storj/StorjBackend.cs new file mode 100644 index 000000000..43b4f42a9 --- /dev/null +++ b/Duplicati/Library/Backend/Storj/StorjBackend.cs @@ -0,0 +1,368 @@ +using Duplicati.Library.Interface; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Linq.Expressions; +using System.Runtime.InteropServices; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using uplink.NET.Interfaces; +using uplink.NET.Models; +using uplink.NET.Services; + +namespace Duplicati.Library.Backend.Storj +{ + public class Storj : IStreamingBackend + { + private const string STORJ_AUTH_METHOD = "storj-auth-method"; + private const string STORJ_SATELLITE = "storj-satellite"; + private const string STORJ_API_KEY = "storj-api-key"; + private const string STORJ_SECRET = "storj-secret"; + private const string STORJ_SHARED_ACCESS = "storj-shared-access"; + private const string STORJ_BUCKET = "storj-bucket"; + private const string STORJ_FOLDER = "storj-folder"; + + private const string PROTOCOL_KEY = "storj"; + private const string STORJ_PARTNER_ID = "duplicati"; + + private readonly string _satellite; + private readonly string _api_key; + private readonly string _secret; + private readonly string _bucket; + private readonly string _folder; + private Access _access; + private IBucketService _bucketService; + private IObjectService _objectService; + + public static readonly Dictionary<string, string> KNOWN_STORJ_SATELLITES = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){ + { "US Central", "us1.storj.io:7777" }, + { "Asia East", "ap1.storj.io:7777" }, + { "Europe", "eu1.storj.io:7777" }, + }; + + public static readonly Dictionary<string, string> KNOWN_AUTHENTICATION_METHODS = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){ + { "API key", "API key" }, + { "Access grant", "Access grant" }, + }; + + [DllImport("kernel32.dll")] + protected static extern IntPtr LoadLibrary(string filename); + + private static bool _libraryLoaded = false; + private static void InitStorjLibrary() + { + if (_libraryLoaded) + return; + + if (Duplicati.Library.Common.Platform.IsClientWindows) //We need to init only on Windows to distinguish between x64 and x86 + { + if (System.Environment.Is64BitProcess) + { + var res = LoadLibrary("win-x64/storj_uplink.dll"); + } + else + { + var res = LoadLibrary("win-x86/storj_uplink.dll"); + } + } + Access.SetTempDirectory(Library.Utility.TempFolder.SystemTempPath); + _libraryLoaded = true; + } + + // ReSharper disable once UnusedMember.Global + // This constructor is needed by the BackendLoader. + public Storj() + { + } + + // ReSharper disable once UnusedMember.Global + // This constructor is needed by the BackendLoader. + public Storj(string url, Dictionary<string, string> options) + { + InitStorjLibrary(); + + foreach(var option in options.ToList()) + { + if(option.Key.ToLower().Contains("tardigrade")) + { + options.Add(option.Key.ToLower().Replace("tardigrade", "storj"), option.Value); + } + } + + var auth_method = options[STORJ_AUTH_METHOD]; + if (auth_method == "Access grant") + { + //Create an access from the access grant + var shared_access = options[STORJ_SHARED_ACCESS]; + _access = new Access(shared_access, new Config() { UserAgent = STORJ_PARTNER_ID }); + } + else + { + //Create an access for a satellite, API key and encryption passphrase + _satellite = options[STORJ_SATELLITE]; + + if (options.ContainsKey(STORJ_API_KEY)) + { + _api_key = options[STORJ_API_KEY]; + } + if (options.ContainsKey(STORJ_SECRET)) + { + _secret = options[STORJ_SECRET]; + } + + _access = new Access(_satellite, _api_key, _secret, new Config() { UserAgent = STORJ_PARTNER_ID }); + } + + _bucketService = new BucketService(_access); + _objectService = new ObjectService(_access); + + //If no bucket was provided use the default "duplicati"-bucket + if (options.ContainsKey(STORJ_BUCKET)) + { + _bucket = options[STORJ_BUCKET]; + } + else + { + _bucket = "duplicati"; + } + + if (options.ContainsKey(STORJ_FOLDER)) + { + _folder = options[STORJ_FOLDER]; + } + } + + public string DisplayName + { + get { return Strings.Storj.DisplayName; } + } + + public string ProtocolKey => PROTOCOL_KEY; + + public IList<ICommandLineArgument> SupportedCommands + { + get + { + return new List<ICommandLineArgument>(new ICommandLineArgument[] { + new CommandLineArgument(STORJ_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, "API key"), + new CommandLineArgument(STORJ_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, "us1.storj.io:7777"), + new CommandLineArgument(STORJ_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong), + new CommandLineArgument(STORJ_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong), + new CommandLineArgument(STORJ_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong), + new CommandLineArgument(STORJ_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong), + new CommandLineArgument(STORJ_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong), + }); + } + } + + public string Description + { + get + { + return Strings.Storj.Description; + } + } + + public string[] DNSName + { + get + { + return new string[0]; + } + } + + public void CreateFolder() + { + //Storj DCS has no folders + } + + public void Delete(string remotename) + { + var deleteTask = DeleteAsync(remotename); + deleteTask.Wait(); + } + + public async Task DeleteAsync(string remotename) + { + try + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename); + } + catch (Exception root) + { + throw new FileMissingException(root); + } + } + + public void Dispose() + { + if (_objectService != null) + { + _objectService = null; + } + if (_bucketService != null) + { + _bucketService = null; + } + if (_access != null) + { + _access.Dispose(); + _access = null; + } + } + + public void Get(string remotename, string filename) + { + var getTask = GetAsync(remotename, filename); + getTask.Wait(); + } + + public async Task GetAsync(string remotename, string filename) + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); + await download.StartDownloadAsync(); + + if (download.Completed) + { + using (FileStream file = new FileStream(filename, FileMode.Create)) + { + await file.WriteAsync(download.DownloadedBytes, 0, (int)download.BytesReceived); + await file.FlushAsync().ConfigureAwait(false); + } + } + } + + public void Get(string remotename, Stream stream) + { + var getTask = GetAsync(remotename, stream); + getTask.Wait(); + } + + public async Task GetAsync(string remotename, Stream stream) + { + int index = 0; + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); + download.DownloadOperationProgressChanged += (op) => + { + int newPartLength = (int)op.BytesReceived - index; + byte[] newPart = new byte[newPartLength]; + Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength); + stream.Write(newPart, 0, newPartLength); + index = index + newPartLength; + }; + await download.StartDownloadAsync(); + } + + public IEnumerable<IFileEntry> List() + { + var listTask = ListAsync(); + listTask.Wait(); + return listTask.Result; + } + + private async Task<IEnumerable<IFileEntry>> ListAsync() + { + List<StorjFile> files = new List<StorjFile>(); + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var prefix = GetBasePath(); + var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }); + + foreach (var obj in objects.Items) + { + StorjFile file = new StorjFile(obj); + if (prefix != "") + { + file.Name = file.Name.Replace(prefix, ""); + } + files.Add(file); + } + + return files; + } + + public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) + { + using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) + await PutAsync(remotename, fs, cancelToken); + } + + public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) + { + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + CustomMetadata custom = new CustomMetadata(); + custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") }); + custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") }); + var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false); + await upload.StartUploadAsync(); + } + + public void Test() + { + var testTask = TestAsync(); + testTask.Wait(10000); + if (!testTask.Result) + { + throw new Exception(Strings.Storj.TestConnectionFailed); + } + } + + /// <summary> + /// Test the connection by: + /// - creating the bucket (if it not already exists) + /// - uploading 256 random bytes to a test-file + /// - downloading the file back and expecting 256 bytes + /// </summary> + /// <returns>true, if the test was successfull or and exception</returns> + private async Task<bool> TestAsync() + { + string testFileName = GetBasePath() + "duplicati_test.dat"; + + var bucket = await _bucketService.EnsureBucketAsync(_bucket); + var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false); + await upload.StartUploadAsync(); + + var download = await _objectService.DownloadObjectAsync(bucket, testFileName, new DownloadOptions(), false); + await download.StartDownloadAsync(); + + await _objectService.DeleteObjectAsync(bucket, testFileName); + + if (download.Failed || download.BytesReceived != 256) + { + throw new Exception(download.ErrorMessage); + } + + return true; + } + + /// <summary> + /// Gets the base path - depending on there is a folder set or not + /// </summary> + /// <returns>The base path within a bucket where the backup shall be placed</returns> + private string GetBasePath() + { + if (!string.IsNullOrEmpty(_folder)) + return _folder + "/"; + else + return ""; + } + + /// <summary> + /// Creates some random bytes with the given length - just for testing the connection + /// </summary> + /// <param name="length">The length of the bytes to create</param> + /// <returns>A byte-array with the given length</returns> + private static byte[] GetRandomBytes(long length) + { + byte[] bytes = new byte[length]; + Random rand = new Random(); + rand.NextBytes(bytes); + + return bytes; + } + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs b/Duplicati/Library/Backend/Storj/StorjConfig.cs index 1464fe17f..7b449645f 100644 --- a/Duplicati/Library/Backend/Tardigrade/TardigradeConfig.cs +++ b/Duplicati/Library/Backend/Storj/StorjConfig.cs @@ -5,12 +5,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace Duplicati.Library.Backend.Tardigrade +namespace Duplicati.Library.Backend.Storj { - public class TardigradeConfig : IWebModule + public class StorjConfig : IWebModule { private const ConfigType DEFAULT_CONFIG_TYPE = ConfigType.Satellites; - private const string KEY_CONFIGTYPE = "tardigrade-config"; + private const string KEY_CONFIGTYPE = "storj-config"; private static readonly string DEFAULT_CONFIG_TYPE_STR = Enum.GetName(typeof(ConfigType), DEFAULT_CONFIG_TYPE); public enum ConfigType @@ -21,11 +21,11 @@ namespace Duplicati.Library.Backend.Tardigrade #region IWebModule implementation - public string Key { get { return "tardigrade-getconfig"; } } + public string Key { get { return "storj-getconfig"; } } - public string DisplayName { get { return "Tardigrade configuration module"; } } + public string DisplayName { get { return "Storj DCS configuration module"; } } - public string Description { get { return "Exposes Tardigrade configuration as a web module"; } } + public string Description { get { return "Exposes Storj DCS configuration as a web module"; } } public IList<ICommandLineArgument> SupportedCommands { @@ -56,11 +56,11 @@ namespace Duplicati.Library.Backend.Tardigrade switch (ct) { case ConfigType.Satellites: - return Tardigrade.KNOWN_TARDIGRADE_SATELLITES; + return Storj.KNOWN_STORJ_SATELLITES; case ConfigType.AuthenticationMethods: - return Tardigrade.KNOWN_AUTHENTICATION_METHODS; + return Storj.KNOWN_AUTHENTICATION_METHODS; default: - return Tardigrade.KNOWN_TARDIGRADE_SATELLITES; + return Storj.KNOWN_STORJ_SATELLITES; } } #endregion diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs b/Duplicati/Library/Backend/Storj/StorjFile.cs index 794e427e0..fa83c3907 100644 --- a/Duplicati/Library/Backend/Tardigrade/TardigradeFile.cs +++ b/Duplicati/Library/Backend/Storj/StorjFile.cs @@ -5,12 +5,12 @@ using System.Linq; using System.Text; using System.Threading.Tasks; -namespace Duplicati.Library.Backend.Tardigrade +namespace Duplicati.Library.Backend.Storj { - public class TardigradeFile : IFileEntry + public class StorjFile : IFileEntry { - public static readonly string TARDIGRADE_LAST_ACCESS = "DUPLICATI:LAST-ACCESS"; - public static readonly string TARDIGRADE_LAST_MODIFICATION = "DUPLICATI:LAST-MODIFICATION"; + public static readonly string STORJ_LAST_ACCESS = "DUPLICATI:LAST-ACCESS"; + public static readonly string STORJ_LAST_MODIFICATION = "DUPLICATI:LAST-MODIFICATION"; public bool IsFolder { get; set; } public DateTime LastAccess { get; set; } @@ -21,15 +21,15 @@ namespace Duplicati.Library.Backend.Tardigrade public long Size { get; set; } - public TardigradeFile() + public StorjFile() { } - public TardigradeFile(uplink.NET.Models.Object tardigradeObject) + public StorjFile(uplink.NET.Models.Object tardigradeObject) { IsFolder = tardigradeObject.IsPrefix; - var lastAccess = tardigradeObject.CustomMetaData.Entries.Where(e => e.Key == TARDIGRADE_LAST_ACCESS).FirstOrDefault(); + var lastAccess = tardigradeObject.CustomMetadata.Entries.Where(e => e.Key == STORJ_LAST_ACCESS).FirstOrDefault(); if (lastAccess != null && !string.IsNullOrEmpty(lastAccess.Value)) { LastAccess = DateTime.Parse(lastAccess.Value); @@ -39,7 +39,7 @@ namespace Duplicati.Library.Backend.Tardigrade LastAccess = DateTime.MinValue; } - var lastMod = tardigradeObject.CustomMetaData.Entries.Where(e => e.Key == TARDIGRADE_LAST_MODIFICATION).FirstOrDefault(); + var lastMod = tardigradeObject.CustomMetadata.Entries.Where(e => e.Key == STORJ_LAST_MODIFICATION).FirstOrDefault(); if (lastMod != null && !string.IsNullOrEmpty(lastMod.Value)) { LastModification = DateTime.Parse(lastMod.Value); @@ -50,7 +50,7 @@ namespace Duplicati.Library.Backend.Tardigrade } Name = tardigradeObject.Key; - Size = tardigradeObject.SystemMetaData.ContentLength; + Size = tardigradeObject.SystemMetadata.ContentLength; } } } diff --git a/Duplicati/Library/Backend/Storj/Strings.cs b/Duplicati/Library/Backend/Storj/Strings.cs new file mode 100644 index 000000000..c1de9425f --- /dev/null +++ b/Duplicati/Library/Backend/Storj/Strings.cs @@ -0,0 +1,30 @@ +using Duplicati.Library.Localization.Short; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Duplicati.Library.Backend.Strings +{ + internal static class Storj + { + public static string DisplayName { get { return LC.L(@"Storj DCS (Decentralized Cloud Storage)"); } } + public static string Description { get { return LC.L(@"This backend can read and write data to the Storj DCS."); } } + public static string TestConnectionFailed { get { return LC.L(@"The connection-test failed."); } } + public static string StorjAuthMethodDescriptionShort { get { return LC.L(@"The authentication method"); } } + public static string StorjAuthMethodDescriptionLong { get { return LC.L(@"The authentication method describes which way to use to connect to the network - either via API key or via an access grant."); } } + public static string StorjSatelliteDescriptionShort { get { return LC.L(@"The satellite"); } } + public static string StorjSatelliteDescriptionLong { get { return LC.L(@"The satellite that keeps track of all metadata. Use a Storj DCS server for high-performance SLA-backed connectivity or use a community server. Or even host your own."); } } + public static string StorjAPIKeyDescriptionShort { get { return LC.L(@"The API key"); } } + public static string StorjAPIKeyDescriptionLong { get { return LC.L(@"The API key grants access to a specific project on your chosen satellite. Head over to the dashboard of your satellite to create one if you do not already have an API key."); } } + public static string StorjSecretDescriptionShort { get { return LC.L(@"The encryption passphrase"); } } + public static string StorjSecretDescriptionLong { get { return LC.L(@"The encryption passphrase is used to encrypt your data before sending it to the Storj network. This passphrase can be the only secret to provide - for Storj you do not necessary need any additional encryption (from Duplicati) in place."); } } + public static string StorjSharedAccessDescriptionShort { get { return LC.L(@"The access grant"); } } + public static string StorjSharedAccessDescriptionLong { get { return LC.L(@"An access grant contains all information in one encrypted string. You may use it instead of a satellite, API key and secret."); } } + public static string StorjBucketDescriptionShort { get { return LC.L(@"The bucket"); } } + public static string StorjBucketDescriptionLong { get { return LC.L(@"The bucket where the backup will reside in."); } } + public static string StorjFolderDescriptionShort { get { return LC.L(@"The folder"); } } + public static string StorjFolderDescriptionLong { get { return LC.L(@"The folder within the bucket where the backup will reside in."); } } + } +} diff --git a/Duplicati/Library/Backend/Tardigrade/libstorj_uplink.dylib b/Duplicati/Library/Backend/Storj/libstorj_uplink.dylib Binary files differindex 4f0190dea..4f8ea2938 100644 --- a/Duplicati/Library/Backend/Tardigrade/libstorj_uplink.dylib +++ b/Duplicati/Library/Backend/Storj/libstorj_uplink.dylib diff --git a/Duplicati/Library/Backend/Tardigrade/libstorj_uplink.so b/Duplicati/Library/Backend/Storj/libstorj_uplink.so Binary files differindex 7c307a134..f4f495271 100644 --- a/Duplicati/Library/Backend/Tardigrade/libstorj_uplink.so +++ b/Duplicati/Library/Backend/Storj/libstorj_uplink.so diff --git a/Duplicati/Library/Backend/Storj/packages.config b/Duplicati/Library/Backend/Storj/packages.config new file mode 100644 index 000000000..983b0e7cc --- /dev/null +++ b/Duplicati/Library/Backend/Storj/packages.config @@ -0,0 +1,13 @@ +<?xml version="1.0" encoding="utf-8"?> +<packages> + <package id="sqlite-net-pcl" version="1.7.335" targetFramework="net471" /> + <package id="SQLitePCLRaw.bundle_green" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.core" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.lib.e_sqlite3" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.provider.dynamic_cdecl" version="2.0.3" targetFramework="net471" /> + <package id="System.Buffers" version="4.4.0" targetFramework="net471" /> + <package id="System.Memory" version="4.5.3" targetFramework="net471" /> + <package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net471" /> + <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net471" /> + <package id="uplink.NET" version="2.7.1604" targetFramework="net471" /> +</packages>
\ No newline at end of file diff --git a/Duplicati/Library/Backend/Tardigrade/win-x64/storj_uplink.dll b/Duplicati/Library/Backend/Storj/win-x64/storj_uplink.dll Binary files differindex 29863308c..dbbca788d 100644 --- a/Duplicati/Library/Backend/Tardigrade/win-x64/storj_uplink.dll +++ b/Duplicati/Library/Backend/Storj/win-x64/storj_uplink.dll diff --git a/Duplicati/Library/Backend/Tardigrade/win-x86/storj_uplink.dll b/Duplicati/Library/Backend/Storj/win-x86/storj_uplink.dll Binary files differindex 6f1cc9e95..bbc697695 100644 --- a/Duplicati/Library/Backend/Tardigrade/win-x86/storj_uplink.dll +++ b/Duplicati/Library/Backend/Storj/win-x86/storj_uplink.dll diff --git a/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj b/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj index 0383b960b..5f7ba3ef7 100644 --- a/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj +++ b/Duplicati/Library/Backend/Tardigrade/Duplicati.Library.Backend.Tardigrade.csproj @@ -4,7 +4,7 @@ <PropertyGroup> <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration> <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform> - <ProjectGuid>{AE035E01-C917-4F13-A35E-78F21C1A2F17}</ProjectGuid> + <ProjectGuid>{9A04CB37-DA72-4008-9703-3AC5191974E9}</ProjectGuid> <OutputType>Library</OutputType> <AppDesignerFolder>Properties</AppDesignerFolder> <RootNamespace>Duplicati.Library.Backend.Tardigrade</RootNamespace> @@ -35,24 +35,50 @@ <WarningLevel>4</WarningLevel> </PropertyGroup> <ItemGroup> + <Reference Include="SQLite-net, Version=1.7.335.0, Culture=neutral, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\sqlite-net-pcl.1.7.335\lib\netstandard2.0\SQLite-net.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.batteries_v2, Version=2.0.3.851, Culture=neutral, PublicKeyToken=8226ea5df37bcae9, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.bundle_green.2.0.3\lib\net461\SQLitePCLRaw.batteries_v2.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.core, Version=2.0.3.851, Culture=neutral, PublicKeyToken=1488e028ca7ab535, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.core.2.0.3\lib\netstandard2.0\SQLitePCLRaw.core.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.nativelibrary, Version=2.0.3.851, Culture=neutral, PublicKeyToken=502ed628492ab262, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.bundle_green.2.0.3\lib\net461\SQLitePCLRaw.nativelibrary.dll</HintPath> + </Reference> + <Reference Include="SQLitePCLRaw.provider.dynamic_cdecl, Version=2.0.3.851, Culture=neutral, PublicKeyToken=b68184102cba0b3b, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\SQLitePCLRaw.provider.dynamic_cdecl.2.0.3\lib\netstandard2.0\SQLitePCLRaw.provider.dynamic_cdecl.dll</HintPath> + </Reference> <Reference Include="System" /> + <Reference Include="System.Buffers, Version=4.0.2.0, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Buffers.4.4.0\lib\netstandard2.0\System.Buffers.dll</HintPath> + </Reference> <Reference Include="System.Core" /> + <Reference Include="System.Memory, Version=4.0.1.1, Culture=neutral, PublicKeyToken=cc7b13ffcd2ddd51, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Memory.4.5.3\lib\netstandard2.0\System.Memory.dll</HintPath> + </Reference> + <Reference Include="System.Numerics" /> + <Reference Include="System.Numerics.Vectors, Version=4.1.3.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Numerics.Vectors.4.4.0\lib\net46\System.Numerics.Vectors.dll</HintPath> + </Reference> + <Reference Include="System.Runtime.CompilerServices.Unsafe, Version=4.0.4.1, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\System.Runtime.CompilerServices.Unsafe.4.5.2\lib\netstandard2.0\System.Runtime.CompilerServices.Unsafe.dll</HintPath> + </Reference> <Reference Include="System.Xml.Linq" /> <Reference Include="System.Data.DataSetExtensions" /> <Reference Include="Microsoft.CSharp" /> <Reference Include="System.Data" /> <Reference Include="System.Net.Http" /> <Reference Include="System.Xml" /> - <Reference Include="uplink.NET, Version=2.3.4.0, Culture=neutral, processorArchitecture=MSIL"> - <HintPath>..\..\..\..\packages\uplink.NET.2.3.4\lib\netstandard2.0\uplink.NET.dll</HintPath> + <Reference Include="uplink.NET, Version=2.7.0.0, Culture=neutral, processorArchitecture=MSIL"> + <HintPath>..\..\..\..\packages\uplink.NET.2.7.1604\lib\netstandard2.0\uplink.NET.dll</HintPath> </Reference> </ItemGroup> <ItemGroup> - <Compile Include="TardigradeFile.cs" /> <Compile Include="Strings.cs" /> <Compile Include="TardigradeBackend.cs" /> <Compile Include="Properties\AssemblyInfo.cs" /> - <Compile Include="TardigradeConfig.cs" /> </ItemGroup> <ItemGroup> <ProjectReference Include="..\..\Common\Duplicati.Library.Common.csproj"> @@ -71,29 +97,20 @@ <Project>{de3e5d4c-51ab-4e5e-bee8-e636cebfba65}</Project> <Name>Duplicati.Library.Utility</Name> </ProjectReference> + <ProjectReference Include="..\Storj\Duplicati.Library.Backend.Storj.csproj"> + <Project>{ae035e01-c917-4f13-a35e-78f21c1a2f17}</Project> + <Name>Duplicati.Library.Backend.Storj</Name> + </ProjectReference> </ItemGroup> <ItemGroup> - <None Include="libstorj_uplink.dylib"> - <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> - </None> - <None Include="libstorj_uplink.so"> - <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> - </None> <None Include="packages.config" /> </ItemGroup> - <ItemGroup> - <Content Include="win-x64\storj_uplink.dll"> - <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> - </Content> - <Content Include="win-x86\storj_uplink.dll"> - <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> - </Content> - </ItemGroup> <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" /> <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> <PropertyGroup> <ErrorText>Dieses Projekt verweist auf mindestens ein NuGet-Paket, das auf diesem Computer fehlt. Verwenden Sie die Wiederherstellung von NuGet-Paketen, um die fehlenden Dateien herunterzuladen. Weitere Informationen finden Sie unter "http://go.microsoft.com/fwlink/?LinkID=322105". Die fehlende Datei ist "{0}".</ErrorText> </PropertyGroup> - <Error Condition="!Exists('..\..\..\..\packages\uplink.NET.2.3.4\build\net40\uplink.NET.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\uplink.NET.2.3.4\build\net40\uplink.NET.targets'))" /> + <Error Condition="!Exists('..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets')" Text="$([System.String]::Format('$(ErrorText)', '..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets'))" /> </Target> -</Project> + <Import Project="..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets" Condition="Exists('..\..\..\..\packages\SQLitePCLRaw.lib.e_sqlite3.2.0.3\build\net461\SQLitePCLRaw.lib.e_sqlite3.targets')" /> +</Project>
\ No newline at end of file diff --git a/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs b/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs index 5f3cc907e..762e1e0c9 100644 --- a/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs +++ b/Duplicati/Library/Backend/Tardigrade/Properties/AssemblyInfo.cs @@ -20,7 +20,7 @@ using System.Runtime.InteropServices; [assembly: ComVisible(false)] // Die folgende GUID bestimmt die ID der Typbibliothek, wenn dieses Projekt für COM verfügbar gemacht wird -[assembly: Guid("ae035e01-c917-4f13-a35e-78f21c1a2f17")] +[assembly: Guid("9A04CB37-DA72-4008-9703-3AC5191974E9")] // Versionsinformationen für eine Assembly bestehen aus den folgenden vier Werten: // diff --git a/Duplicati/Library/Backend/Tardigrade/Strings.cs b/Duplicati/Library/Backend/Tardigrade/Strings.cs index 7540c2859..3703872f2 100644 --- a/Duplicati/Library/Backend/Tardigrade/Strings.cs +++ b/Duplicati/Library/Backend/Tardigrade/Strings.cs @@ -9,22 +9,7 @@ namespace Duplicati.Library.Backend.Strings { internal static class Tardigrade { - public static string DisplayName { get { return LC.L(@"Tardigrade Decentralized Cloud Storage"); } } - public static string Description { get { return LC.L(@"This backend can read and write data to the Tardigrade Decentralized Cloud Storage."); } } - public static string TestConnectionFailed { get { return LC.L(@"The connection-test failed."); } } - public static string TardigradeAuthMethodDescriptionShort { get { return LC.L(@"The authentication method"); } } - public static string TardigradeAuthMethodDescriptionLong { get { return LC.L(@"The authentication method describes which way to use to connect to the network - either via API key or via an access grant."); } } - public static string TardigradeSatelliteDescriptionShort { get { return LC.L(@"The satellite"); } } - public static string TardigradeSatelliteDescriptionLong { get { return LC.L(@"The satellite that keeps track of all metadata. Use a Tardigrade-grade server for high-performance SLA-backed connectivity or use a community server. Or even host your own."); } } - public static string TardigradeAPIKeyDescriptionShort { get { return LC.L(@"The API key"); } } - public static string TardigradeAPIKeyDescriptionLong { get { return LC.L(@"The API key grants access to a specific project on your chosen satellite. Head over to the dashboard of your satellite to create one if you do not already have an API key."); } } - public static string TardigradeSecretDescriptionShort { get { return LC.L(@"The encryption passphrase"); } } - public static string TardigradeSecretDescriptionLong { get { return LC.L(@"The encryption passphrase is used to encrypt your data before sending it to the tardigrade network. This passphrase can be the only secret to provide - for Tardigrade you do not necessary need any additional encryption (from Duplicati) in place."); } } - public static string TardigradeSharedAccessDescriptionShort { get { return LC.L(@"The access grant"); } } - public static string TardigradeSharedAccessDescriptionLong { get { return LC.L(@"An access grant contains all information in one encrypted string. You may use it instead of a satellite, API key and secret."); } } - public static string TardigradeBucketDescriptionShort { get { return LC.L(@"The bucket"); } } - public static string TardigradeBucketDescriptionLong { get { return LC.L(@"The bucket where the backup will reside in."); } } - public static string TardigradeFolderDescriptionShort { get { return LC.L(@"The folder"); } } - public static string TardigradeFolderDescriptionLong { get { return LC.L(@"The folder within the bucket where the backup will reside in."); } } + public static string DisplayName { get { return LC.L(@"Tardigrade Decentralised Cloud Storage (Deprecated)"); } } + public static string Description { get { return LC.L(@"This backend can read and write data to the Tardigrade Decentralized Cloud Storage. It is deprecated - please move over to the new Storj DCS. (Deprecated)"); } } } } diff --git a/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs b/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs index 0bf808105..255830500 100644 --- a/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs +++ b/Duplicati/Library/Backend/Tardigrade/TardigradeBackend.cs @@ -14,7 +14,15 @@ using uplink.NET.Services; namespace Duplicati.Library.Backend.Tardigrade { - public class Tardigrade : IStreamingBackend + + /// <summary> + /// This backend is deprecated! It will be removed in the future. + /// Tardigrade renamed to Storj DCS in Spring 2021 - but existing Tardigrade-Configurations could not be easily renamed. + /// So we decided to "copy" Tardigrade over to the new name Storj DCS. In order to reduce duplicate code, the old + /// Tardigrade-Backend hands it's logic over to Storj DCS. Only the UI-specific part, the config-parameters and + /// the protocol-key stay here named for Tardigrade. + /// </summary> + public class Tardigrade : Duplicati.Library.Backend.Storj.Storj, IStreamingBackend { private const string TARDIGRADE_AUTH_METHOD = "tardigrade-auth-method"; private const string TARDIGRADE_SATELLITE = "tardigrade-satellite"; @@ -23,339 +31,50 @@ namespace Duplicati.Library.Backend.Tardigrade private const string TARDIGRADE_SHARED_ACCESS = "tardigrade-shared-access"; private const string TARDIGRADE_BUCKET = "tardigrade-bucket"; private const string TARDIGRADE_FOLDER = "tardigrade-folder"; - private const string PROTOCOL_KEY = "tardigrade"; - private const string TARDIGRADE_PARTNER_ID = "duplicati"; - - private readonly string _satellite; - private readonly string _api_key; - private readonly string _secret; - private readonly string _bucket; - private readonly string _folder; - private Access _access; - private IBucketService _bucketService; - private IObjectService _objectService; - - public static readonly Dictionary<string, string> KNOWN_TARDIGRADE_SATELLITES = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){ - { "US Central 1", "us-central-1.tardigrade.io:7777" }, - { "Asia East 1", "asia-east-1.tardigrade.io:7777" }, - { "Saltlake", "saltlake.tardigrade.io:7777" }, - { "Europe West 1", "europe-west-1.tardigrade.io:7777" }, - { "Europe North 1", "europe-north-1.tardigrade.io:7777" }, - }; - - public static readonly Dictionary<string, string> KNOWN_AUTHENTICATION_METHODS = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){ - { "API key", "API key" }, - { "Access grant", "Access grant" }, - }; - - [DllImport("kernel32.dll")] - protected static extern IntPtr LoadLibrary(string filename); - - private static bool _libraryLoaded = false; - private static void InitStorjLibrary() - { - if (_libraryLoaded) - return; - if (Duplicati.Library.Common.Platform.IsClientWindows) //We need to init only on Windows to distinguish between x64 and x86 - { - if (System.Environment.Is64BitProcess) - { - var res = LoadLibrary("win-x64/storj_uplink.dll"); - } - else - { - var res = LoadLibrary("win-x86/storj_uplink.dll"); - } - } - Access.SetTempDirectory(Library.Utility.TempFolder.SystemTempPath); - _libraryLoaded = true; - } + private const string PROTOCOL_KEY = "tardigrade"; // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public Tardigrade() + public Tardigrade():base() { } // ReSharper disable once UnusedMember.Global // This constructor is needed by the BackendLoader. - public Tardigrade(string url, Dictionary<string, string> options) + public Tardigrade(string url, Dictionary<string, string> options) :base(url, options) { - InitStorjLibrary(); - - var auth_method = options[TARDIGRADE_AUTH_METHOD]; - if (auth_method == "Access grant") - { - //Create an access from the access grant - var shared_access = options[TARDIGRADE_SHARED_ACCESS]; - _access = new Access(shared_access, new Config() { UserAgent = TARDIGRADE_PARTNER_ID }); - } - else - { - //Create an access for a satellite, API key and encryption passphrase - _satellite = options[TARDIGRADE_SATELLITE]; - - if (options.ContainsKey(TARDIGRADE_API_KEY)) - { - _api_key = options[TARDIGRADE_API_KEY]; - } - if (options.ContainsKey(TARDIGRADE_SECRET)) - { - _secret = options[TARDIGRADE_SECRET]; - } - - _access = new Access(_satellite, _api_key, _secret, new Config() { UserAgent = TARDIGRADE_PARTNER_ID }); - } - - _bucketService = new BucketService(_access); - _objectService = new ObjectService(_access); - - //If no bucket was provided use the default "duplicati"-bucket - if (options.ContainsKey(TARDIGRADE_BUCKET)) - { - _bucket = options[TARDIGRADE_BUCKET]; - } - else - { - _bucket = "duplicati"; - } - - if (options.ContainsKey(TARDIGRADE_FOLDER)) - { - _folder = options[TARDIGRADE_FOLDER]; - } } - public string DisplayName + public new string DisplayName { get { return Strings.Tardigrade.DisplayName; } } - public string ProtocolKey => PROTOCOL_KEY; + public new string ProtocolKey => PROTOCOL_KEY; - public IList<ICommandLineArgument> SupportedCommands + public new IList<ICommandLineArgument> SupportedCommands { get { return new List<ICommandLineArgument>(new ICommandLineArgument[] { - new CommandLineArgument(TARDIGRADE_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeAuthMethodDescriptionShort, Strings.Tardigrade.TardigradeAuthMethodDescriptionLong, "API key"), - new CommandLineArgument(TARDIGRADE_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeSatelliteDescriptionShort, Strings.Tardigrade.TardigradeSatelliteDescriptionLong, "us-central-1.tardigrade.io:7777"), - new CommandLineArgument(TARDIGRADE_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeAPIKeyDescriptionShort, Strings.Tardigrade.TardigradeAPIKeyDescriptionLong), - new CommandLineArgument(TARDIGRADE_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Tardigrade.TardigradeSecretDescriptionShort, Strings.Tardigrade.TardigradeSecretDescriptionLong), - new CommandLineArgument(TARDIGRADE_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeSharedAccessDescriptionShort, Strings.Tardigrade.TardigradeSharedAccessDescriptionLong), - new CommandLineArgument(TARDIGRADE_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeBucketDescriptionShort, Strings.Tardigrade.TardigradeBucketDescriptionLong), - new CommandLineArgument(TARDIGRADE_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Tardigrade.TardigradeFolderDescriptionShort, Strings.Tardigrade.TardigradeFolderDescriptionLong), + new CommandLineArgument(TARDIGRADE_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, "API key", null, null), + new CommandLineArgument(TARDIGRADE_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, "us1.storj.io:7777", null, null), + new CommandLineArgument(TARDIGRADE_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong, null, null, null), + new CommandLineArgument(TARDIGRADE_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong, null, null, null), + new CommandLineArgument(TARDIGRADE_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong, null, null, null), + new CommandLineArgument(TARDIGRADE_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong, null, null, null), + new CommandLineArgument(TARDIGRADE_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong, null, null, null), }); } } - public string Description + public new string Description { get { return Strings.Tardigrade.Description; } } - - public string[] DNSName - { - get - { - return new string[0]; - } - } - - public void CreateFolder() - { - //Tardigrade has no folders - } - - public void Delete(string remotename) - { - var deleteTask = DeleteAsync(remotename); - deleteTask.Wait(); - } - - public async Task DeleteAsync(string remotename) - { - try - { - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename); - } - catch (Exception root) - { - throw new FileMissingException(root); - } - } - - public void Dispose() - { - if (_objectService != null) - { - _objectService = null; - } - if (_bucketService != null) - { - _bucketService = null; - } - if (_access != null) - { - _access.Dispose(); - _access = null; - } - } - - public void Get(string remotename, string filename) - { - var getTask = GetAsync(remotename, filename); - getTask.Wait(); - } - - public async Task GetAsync(string remotename, string filename) - { - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); - await download.StartDownloadAsync(); - - if (download.Completed) - { - using (FileStream file = new FileStream(filename, FileMode.Create)) - { - await file.WriteAsync(download.DownloadedBytes, 0, (int)download.BytesReceived); - await file.FlushAsync().ConfigureAwait(false); - } - } - } - - public void Get(string remotename, Stream stream) - { - var getTask = GetAsync(remotename, stream); - getTask.Wait(); - } - - public async Task GetAsync(string remotename, Stream stream) - { - int index = 0; - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false); - download.DownloadOperationProgressChanged += (op) => - { - int newPartLength = (int)op.BytesReceived - index; - byte[] newPart = new byte[newPartLength]; - Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength); - stream.Write(newPart, 0, newPartLength); - index = index + newPartLength; - }; - await download.StartDownloadAsync(); - } - - public IEnumerable<IFileEntry> List() - { - var listTask = ListAsync(); - listTask.Wait(); - return listTask.Result; - } - - private async Task<IEnumerable<IFileEntry>> ListAsync() - { - List<TardigradeFile> files = new List<TardigradeFile>(); - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - var prefix = GetBasePath(); - var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix }); - - foreach (var obj in objects.Items) - { - TardigradeFile file = new TardigradeFile(obj); - if (prefix != "") - { - file.Name = file.Name.Replace(prefix, ""); - } - files.Add(file); - } - - return files; - } - - public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken) - { - using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read)) - await PutAsync(remotename, fs, cancelToken); - } - - public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken) - { - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - CustomMetadata custom = new CustomMetadata(); - custom.Entries.Add(new CustomMetadataEntry { Key = TardigradeFile.TARDIGRADE_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") }); - custom.Entries.Add(new CustomMetadataEntry { Key = TardigradeFile.TARDIGRADE_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") }); - var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false); - await upload.StartUploadAsync(); - } - - public void Test() - { - var testTask = TestAsync(); - testTask.Wait(10000); - if (!testTask.Result) - { - throw new Exception(Strings.Tardigrade.TestConnectionFailed); - } - } - - /// <summary> - /// Test the connection by: - /// - creating the bucket (if it not already exists) - /// - uploading 256 random bytes to a test-file - /// - downloading the file back and expecting 256 bytes - /// </summary> - /// <returns>true, if the test was successfull or and exception</returns> - private async Task<bool> TestAsync() - { - string testFileName = GetBasePath() + "duplicati_test.dat"; - - var bucket = await _bucketService.EnsureBucketAsync(_bucket); - var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false); - await upload.StartUploadAsync(); - - var download = await _objectService.DownloadObjectAsync(bucket, testFileName, new DownloadOptions(), false); - await download.StartDownloadAsync(); - - await _objectService.DeleteObjectAsync(bucket, testFileName); - - if (download.Failed || download.BytesReceived != 256) - { - throw new Exception(download.ErrorMessage); - } - - return true; - } - - /// <summary> - /// Gets the base path - depending on there is a folder set or not - /// </summary> - /// <returns>The base path within a bucket where the backup shall be placed</returns> - private string GetBasePath() - { - if (!string.IsNullOrEmpty(_folder)) - return _folder + "/"; - else - return ""; - } - - /// <summary> - /// Creates some random bytes with the given length - just for testing the connection - /// </summary> - /// <param name="length">The length of the bytes to create</param> - /// <returns>A byte-array with the given length</returns> - private static byte[] GetRandomBytes(long length) - { - byte[] bytes = new byte[length]; - Random rand = new Random(); - rand.NextBytes(bytes); - - return bytes; - } } } diff --git a/Duplicati/Library/Backend/Tardigrade/packages.config b/Duplicati/Library/Backend/Tardigrade/packages.config index 51d5eec31..983b0e7cc 100644 --- a/Duplicati/Library/Backend/Tardigrade/packages.config +++ b/Duplicati/Library/Backend/Tardigrade/packages.config @@ -1,4 +1,13 @@ <?xml version="1.0" encoding="utf-8"?> <packages> - <package id="uplink.NET" version="2.3.4" targetFramework="net471" /> + <package id="sqlite-net-pcl" version="1.7.335" targetFramework="net471" /> + <package id="SQLitePCLRaw.bundle_green" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.core" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.lib.e_sqlite3" version="2.0.3" targetFramework="net471" /> + <package id="SQLitePCLRaw.provider.dynamic_cdecl" version="2.0.3" targetFramework="net471" /> + <package id="System.Buffers" version="4.4.0" targetFramework="net471" /> + <package id="System.Memory" version="4.5.3" targetFramework="net471" /> + <package id="System.Numerics.Vectors" version="4.4.0" targetFramework="net471" /> + <package id="System.Runtime.CompilerServices.Unsafe" version="4.5.2" targetFramework="net471" /> + <package id="uplink.NET" version="2.7.1604" targetFramework="net471" /> </packages>
\ No newline at end of file diff --git a/Duplicati/Library/Main/Controller.cs b/Duplicati/Library/Main/Controller.cs index 18669a994..5b319fab3 100644 --- a/Duplicati/Library/Main/Controller.cs +++ b/Duplicati/Library/Main/Controller.cs @@ -816,8 +816,12 @@ namespace Duplicati.Library.Main if (string.Equals(m_options.CompressionModule, "7z", StringComparison.OrdinalIgnoreCase))
Logging.Log.WriteWarningMessage(LOGTAG, "7zModuleHasIssues", null, "The 7z compression module has known issues and should only be used for experimental purposes");
- //TODO: Based on the action, see if all options are relevant
- }
+ //Inform the user about the deprecated Tardigrade-Backend. They should switch to Storj DCS instead.
+ if (string.Equals(new Library.Utility.Uri(m_backend).Scheme, "tardigrade", StringComparison.OrdinalIgnoreCase))
+ Logging.Log.WriteWarningMessage(LOGTAG, "TardigradeRename", null, "The Tardigrade-backend got renamed to Storj DCS - please migrate your backups to the new configuration by changing the destination storage type to Storj DCS.");
+
+ //TODO: Based on the action, see if all options are relevant
+ }
/// <summary>
/// Helper method that expands the users chosen source input paths,
diff --git a/Duplicati/Server/Duplicati.Server.csproj b/Duplicati/Server/Duplicati.Server.csproj index 4a81dbdad..86d62d643 100644 --- a/Duplicati/Server/Duplicati.Server.csproj +++ b/Duplicati/Server/Duplicati.Server.csproj @@ -203,8 +203,12 @@ <Project>{C0270709-2A40-43B5-8CF1-69581B9FA2A1}</Project>
<Name>Duplicati.Library.Backend.TahoeLAFS</Name>
</ProjectReference>
- <ProjectReference Include="..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <ProjectReference Include="..\Library\Backend\Storj\Duplicati.Library.Backend.Storj.csproj">
<Project>{ae035e01-c917-4f13-a35e-78f21c1a2f17}</Project>
+ <Name>Duplicati.Library.Backend.Storj</Name>
+ </ProjectReference>
+ <ProjectReference Include="..\Library\Backend\Tardigrade\Duplicati.Library.Backend.Tardigrade.csproj">
+ <Project>{9a04cb37-da72-4008-9703-3ac5191974e9}</Project>
<Name>Duplicati.Library.Backend.Tardigrade</Name>
</ProjectReference>
<ProjectReference Include="..\Library\Backend\TencentCOS\Duplicati.Library.Backend.TencentCOS.csproj">
diff --git a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js index 5ed824925..63a94b64d 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/EditUriBuiltins.js @@ -29,8 +29,9 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, EditUriBackendConfig.templates['box'] = 'templates/backends/oauth.html'; EditUriBackendConfig.templates['dropbox'] = 'templates/backends/oauth.html'; EditUriBackendConfig.templates['sia'] = 'templates/backends/sia.html'; + EditUriBackendConfig.templates['storj'] = 'templates/backends/storj.html'; EditUriBackendConfig.templates['tardigrade'] = 'templates/backends/tardigrade.html'; - EditUriBackendConfig.templates['rclone'] = 'templates/backends/rclone.html'; + EditUriBackendConfig.templates['rclone'] = 'templates/backends/rclone.html'; EditUriBackendConfig.templates['cos'] = 'templates/backends/cos.html'; EditUriBackendConfig.testers['s3'] = function(scope, callback) { @@ -157,21 +158,47 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, scope.s3_client_options = s3_client_options; }; + EditUriBackendConfig.loaders['storj'] = function (scope) { + if (scope.storj_satellites == null) { + AppService.post('/webmodule/storj-getconfig', {'storj-config': 'Satellites'}).then(function (data) { + scope.storj_satellites = data.data.Result; + if (scope.storj_satellite == undefined && scope.storj_satellite_custom == undefined) + scope.storj_satellite = 'us1.storj.io:7777'; + + }, AppUtils.connectionError); + } else { + if (scope.storj_satellite == undefined && scope.storj_satellite_custom == undefined) + scope.storj_satellite = 'us1.storj.io:7777'; + } + + if (scope.storj_auth_methods == null) { + AppService.post('/webmodule/storj-getconfig', {'storj-config': 'AuthenticationMethods'}).then(function (data) { + scope.storj_auth_methods = data.data.Result; + if (scope.storj_auth_method == undefined) + scope.storj_auth_method = 'API key'; + + }, AppUtils.connectionError); + } else { + if (scope.storj_auth_method == undefined) + scope.storj_auth_method = 'API key'; + } + }; + EditUriBackendConfig.loaders['tardigrade'] = function (scope) { if (scope.tardigrade_satellites == null) { - AppService.post('/webmodule/tardigrade-getconfig', {'tardigrade-config': 'Satellites'}).then(function (data) { + AppService.post('/webmodule/storj-getconfig', {'storj-config': 'Satellites'}).then(function (data) { scope.tardigrade_satellites = data.data.Result; if (scope.tardigrade_satellite == undefined && scope.tardigrade_satellite_custom == undefined) - scope.tardigrade_satellite = 'us-central-1.tardigrade.io:7777'; + scope.tardigrade_satellite = 'us1.storj.io:7777'; }, AppUtils.connectionError); } else { if (scope.tardigrade_satellite == undefined && scope.tardigrade_satellite_custom == undefined) - scope.tardigrade_satellite = 'us-central-1.tardigrade.io:7777'; + scope.tardigrade_satellite = 'us1.storj.io:7777'; } if (scope.tardigrade_auth_methods == null) { - AppService.post('/webmodule/tardigrade-getconfig', {'tardigrade-config': 'AuthenticationMethods'}).then(function (data) { + AppService.post('/webmodule/storj-getconfig', {'storj-config': 'AuthenticationMethods'}).then(function (data) { scope.tardigrade_auth_methods = data.data.Result; if (scope.tardigrade_auth_method == undefined) scope.tardigrade_auth_method = 'API key'; @@ -486,7 +513,30 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, delete options[nukeopts[x]]; } - EditUriBackendConfig.parsers['tardigrade'] = function (scope, module, server, port, path, options) { + EditUriBackendConfig.parsers['storj'] = function (scope, module, server, port, path, options) { + if (options['--storj-auth-method']) + scope.storj_auth_method = options['--storj-auth-method']; + if (options['--storj-satellite']) + scope.storj_satellite = options['--storj-satellite']; + if (options['--storj-api-key']) + scope.storj_api_key = options['--storj-api-key']; + if (options['--storj-secret']) + scope.storj_secret = options['--storj-secret']; + if (options['--storj-secret-verify']) + scope.storj_secret_verify = options['--storj-secret-verify']; + if (options['--storj-shared-access']) + scope.storj_shared_access = options['--storj-shared-access']; + if (options['--storj-bucket']) + scope.storj_bucket = options['--storj-bucket']; + if (options['--storj-folder']) + scope.storj_folder = options['--storj-folder']; + + var nukeopts = ['--storj-auth-method','--storj-satellite', '--storj-api-key', '--storj-secret', '--storj-secret-verify', '--storj-shared-access', '--storj-bucket', '--storj-folder']; + for (var x in nukeopts) + delete options[nukeopts[x]]; + }; + + EditUriBackendConfig.parsers['tardigrade'] = function (scope, module, server, port, path, options) { if (options['--tardigrade-auth-method']) scope.tardigrade_auth_method = options['--tardigrade-auth-method']; if (options['--tardigrade-satellite']) @@ -495,14 +545,16 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, scope.tardigrade_api_key = options['--tardigrade-api-key']; if (options['--tardigrade-secret']) scope.tardigrade_secret = options['--tardigrade-secret']; - if (options['--tardigrade-shared-access']) + if (options['--tardigrade-secret-verify']) + scope.tardigrade_secret_verify = options['--tardigrade-secret-verify']; + if (options['--tardigrade-shared-access']) scope.tardigrade_shared_access = options['--tardigrade-shared-access']; - if (options['--tardigrade-bucket']) + if (options['--tardigrade-bucket']) scope.tardigrade_bucket = options['--tardigrade-bucket']; - if (options['--tardigrade-folder']) + if (options['--tardigrade-folder']) scope.tardigrade_folder = options['--tardigrade-folder']; - - var nukeopts = ['--tardigrade-auth-method','--tardigrade-satellite', '--tardigrade-api-key', '--tardigrade-secret', '--tardigrade-shared-access', '--tardigrade-bucket', '--tardigrade-folder']; + + var nukeopts = ['--tardigrade-auth-method','--tardigrade-satellite', '--tardigrade-api-key', '--tardigrade-secret', '--tardigrade-secret-verify', '--tardigrade-shared-access', '--tardigrade-bucket', '--tardigrade-folder']; for (var x in nukeopts) delete options[nukeopts[x]]; }; @@ -743,15 +795,36 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, return url; } + EditUriBackendConfig.builders['storj'] = function (scope) { + var opts = { + 'storj-auth-method': scope.storj_auth_method, + 'storj-satellite': scope.storj_satellite, + 'storj-api-key': scope.storj_api_key, + 'storj-secret': scope.storj_secret, + 'storj-shared-access': scope.storj_shared_access, + 'storj-bucket': scope.storj_bucket, + 'storj-folder': scope.storj_folder + }; + + EditUriBackendConfig.merge_in_advanced_options(scope, opts); + + var url = AppUtils.format('{0}://storj.io/config{1}', + scope.Backend.Key, + AppUtils.encodeDictAsUrl(opts) + ); + + return url; + }; + EditUriBackendConfig.builders['tardigrade'] = function (scope) { var opts = { - 'tardigrade-auth-method': scope.tardigrade_auth_method, + 'tardigrade-auth-method': scope.tardigrade_auth_method, 'tardigrade-satellite': scope.tardigrade_satellite, 'tardigrade-api-key': scope.tardigrade_api_key, 'tardigrade-secret': scope.tardigrade_secret, 'tardigrade-shared-access': scope.tardigrade_shared_access, - 'tardigrade-bucket': scope.tardigrade_bucket, - 'tardigrade-folder': scope.tardigrade_folder + 'tardigrade-bucket': scope.tardigrade_bucket, + 'tardigrade-folder': scope.tardigrade_folder }; EditUriBackendConfig.merge_in_advanced_options(scope, opts); @@ -1110,7 +1183,71 @@ backupApp.service('EditUriBuiltins', function (AppService, AppUtils, SystemInfo, continuation(); }; + EditUriBackendConfig.validaters['storj'] = function (scope, continuation) { + var res = true; + + if(res && !scope['storj_auth_method']){ + res = EditUriBackendConfig.require_field(scope, 'storj_auth_method', gettextCatalog.getString('Authentication method')); + } + + if(res && scope['storj_auth_method'] == 'Access grant'){ + res = EditUriBackendConfig.require_field(scope, 'storj_shared_access', gettextCatalog.getString('storj_shared_access')) && + EditUriBackendConfig.require_field(scope, 'storj_bucket', gettextCatalog.getString('Bucket')); + } + + if(res && scope['storj_auth_method'] == 'API key'){ + res = EditUriBackendConfig.require_field(scope, 'storj_api_key', gettextCatalog.getString('API key')) && + EditUriBackendConfig.require_field(scope, 'storj_secret', gettextCatalog.getString('Encryption passphrase')) && + EditUriBackendConfig.require_field(scope, 'storj_bucket', gettextCatalog.getString('Bucket')); + } + + if(res && scope['storj_auth_method'] == 'API key' && !scope['storj_satellite']){ + res = EditUriBackendConfig.require_field(scope, 'storj_satellite_custom', gettextCatalog.getString('Custom Satellite')); + } + + if(res && scope['storj_auth_method'] == 'API key' && scope['storj_secret'] != scope['storj_secret_verify']) + res = EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('The encryption passphrases do not match')); + + var re = new RegExp('^([a-z0-9]+([a-z0-9\-][a-z0-9])*)+(.[a-z0-9]+([a-z0-9\-][a-z0-9])*)*$'); + if(res && scope['storj_bucket'] && (!re.test(scope['storj_bucket']) || !(scope['storj_bucket'].length > 2 && scope['storj_bucket'].length < 64))){ + res = EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('Bucket name can only be between 3 and 63 characters long and contain only lower-case characters, numbers, periods and dashes')); + } + + if (res) + continuation(); + }; + EditUriBackendConfig.validaters['tardigrade'] = function (scope, continuation) { + var res = true; + + if(res && !scope['tardigrade_auth_method']){ + res = EditUriBackendConfig.require_field(scope, 'tardigrade_auth_method', gettextCatalog.getString('Authentication method')); + } + + if(res && scope['tardigrade_auth_method'] == 'Access grant'){ + res = EditUriBackendConfig.require_field(scope, 'tardigrade_shared_access', gettextCatalog.getString('tardigrade_shared_access')) && + EditUriBackendConfig.require_field(scope, 'tardigrade_bucket', gettextCatalog.getString('Bucket')); + } + + if(res && scope['tardigrade_auth_method'] == 'API key'){ + res = EditUriBackendConfig.require_field(scope, 'tardigrade_api_key', gettextCatalog.getString('API key')) && + EditUriBackendConfig.require_field(scope, 'tardigrade_secret', gettextCatalog.getString('Encryption passphrase')) && + EditUriBackendConfig.require_field(scope, 'tardigrade_bucket', gettextCatalog.getString('Bucket')); + } + + if(res && scope['tardigrade_auth_method'] == 'API key' && !scope['tardigrade_satellite']){ + res = EditUriBackendConfig.require_field(scope, 'tardigrade_satellite_custom', gettextCatalog.getString('Custom Satellite')); + } + + if(res && scope['tardigrade_auth_method'] == 'API key' && scope['tardigrade_secret'] != scope['tardigrade_secret_verify']) + res = EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('The encryption passphrases do not match')); + + var re = new RegExp('^([a-z0-9]+([a-z0-9\-][a-z0-9])*)+(.[a-z0-9]+([a-z0-9\-][a-z0-9])*)*$'); + if(res && scope['tardigrade_bucket'] && (!re.test(scope['tardigrade_bucket']) || !(scope['tardigrade_bucket'].length > 2 && scope['tardigrade_bucket'].length < 64))){ + res = EditUriBackendConfig.show_error_dialog(gettextCatalog.getString('Bucket name can only be between 3 and 63 characters long and contain only lower-case characters, numbers, periods and dashes')); + } + + if (res) continuation(); }; diff --git a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js index c422020e5..b6b4e4e52 100644 --- a/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js +++ b/Duplicati/Server/webroot/ngax/scripts/services/SystemInfo.js @@ -71,6 +71,7 @@ backupApp.service('SystemInfo', function($rootScope, $timeout, $cookies, AppServ 'mssp': null, 'dropbox': null, 'sia': null, + 'storj': null, 'tardigrade': null, 'jottacloud': null, 'rclone': null, diff --git a/Duplicati/Server/webroot/ngax/templates/backends/storj.html b/Duplicati/Server/webroot/ngax/templates/backends/storj.html new file mode 100644 index 000000000..bb25db55a --- /dev/null +++ b/Duplicati/Server/webroot/ngax/templates/backends/storj.html @@ -0,0 +1,43 @@ +<div class="input select"> + <label for="storj_auth_method" translate>Authentication method</label> + <select name="storj_auth_method" id="storj_auth_method" ng-model="$parent.storj_auth_method" + ng-options="v as k + ' (' + v + ')' for (k, v) in storj_auth_methods | orderBy: k"> + <option value="" translate >Authentication method ({{auth_method}}) + </option> + </select> +</div> +<div class="input select" ng-show="$parent.storj_auth_method == 'API key'"> + <label for="storj_satellite" translate>Satellite</label> + <select name="storj_satellite" id="storj_satellite" ng-model="$parent.storj_satellite" + ng-options="v as k + ' (' + v + ')' for (k, v) in storj_satellites | orderBy: k"> + <option value="" translate translate-params-satellite="storj_satellite_custom || ''">Custom Satellite ({{satellite}}) + </option> + </select> + + <input ng-hide="contains_value(storj_satellites, storj_satellite)" type="text" id="storj_satellite_custom" + ng-model="$parent.storj_satellite_custom" placeholder="{{'Custom Satellite' | translate}}"/> +</div> +<div class="input text" ng-show="$parent.storj_auth_method == 'API key'"> + <label for="storj_api_key" translate>API key</label> + <input type="text" name="storj_api_key" id="storj_api_key" ng-model="$parent.storj_api_key" placeholder="{{'The API key' | translate}}" /> +</div> +<div class="input password" ng-show="$parent.storj_auth_method == 'API key'"> + <label for="storj_secret" translate>Encryption passphrase</label> + <input autocomplete="new-password" type="password" name="storj_secret" id="storj_secret" ng-model="$parent.storj_secret" placeholder="{{'The encryption passphrase' | translate}}" /> +</div> +<div class="input password" ng-show="$parent.storj_auth_method == 'API key'"> + <label for="storj_secret_verify" translate>Verify encryption passphrase</label> + <input autocomplete="new-password" type="password" name="storj_secret_verify" id="storj_secret_verify" ng-model="$parent.storj_secret_verify" placeholder="{{'The encryption passphrase (for verification)' | translate}}" /> +</div> +<div class="input text" ng-show="$parent.storj_auth_method == 'Access grant'"> + <label for="storj_shared_access" translate>Access grant</label> + <input type="text" name="storj_shared_access" id="storj_shared_access" ng-model="$parent.storj_shared_access" placeholder="{{'The access grant instead of the info above'}}" /> +</div> +<div class="input text"> + <label for="storj_bucket" translate>Bucket</label> + <input type="text" name="storj_bucket" id="storj_bucket" ng-model="$parent.storj_bucket" placeholder="{{'The bucket for storing the backup'}}" /> +</div> +<div class="input text"> + <label for="storj_folder" translate>Folder path</label> + <input type="text" name="storj_folder" id="storj_folder" ng-model="$parent.storj_folder" placeholder="{{'The folder within the bucket for storing the backup'}}" /> +</div> diff --git a/Duplicati/Server/webroot/ngax/templates/backends/tardigrade.html b/Duplicati/Server/webroot/ngax/templates/backends/tardigrade.html index eaa094c69..0131cab56 100644 --- a/Duplicati/Server/webroot/ngax/templates/backends/tardigrade.html +++ b/Duplicati/Server/webroot/ngax/templates/backends/tardigrade.html @@ -25,6 +25,10 @@ <label for="tardigrade_secret" translate>Encryption passphrase</label> <input autocomplete="new-password" type="password" name="tardigrade_secret" id="tardigrade_secret" ng-model="$parent.tardigrade_secret" placeholder="{{'The encryption passphrase' | translate}}" /> </div> +<div class="input password" ng-show="$parent.tardigrade_auth_method == 'API key'"> + <label for="tardigrade_secret_verify" translate>Verify encryption passphrase</label> + <input autocomplete="new-password" type="password" name="tardigrade_secret_verify" id="tardigrade_secret_verify" ng-model="$parent.tardigrade_secret_verify" placeholder="{{'The encryption passphrase (for verification)' | translate}}" /> +</div> <div class="input text" ng-show="$parent.tardigrade_auth_method == 'Access grant'"> <label for="tardigrade_shared_access" translate>Access grant</label> <input type="text" name="tardigrade_shared_access" id="tardigrade_shared_access" ng-model="$parent.tardigrade_shared_access" placeholder="{{'The access grant instead of the info above'}}" /> |