path: root/Duplicati/Library/Backend/Storj/StorjBackend.cs

using Duplicati.Library.Interface;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using uplink.NET.Interfaces;
using uplink.NET.Models;
using uplink.NET.Services;

namespace Duplicati.Library.Backend.Tardigrade
{
    public class Tardigrade : IStreamingBackend
    {
        #region Deprecated parameters - Should be removed in one of the next duplicati versions
        private const string TARDIGRADE_AUTH_METHOD = "tardigrade-auth-method";
        private const string TARDIGRADE_SATELLITE = "tardigrade-satellite";
        private const string TARDIGRADE_API_KEY = "tardigrade-api-key";
        private const string TARDIGRADE_SECRET = "tardigrade-secret";
        private const string TARDIGRADE_SHARED_ACCESS = "tardigrade-shared-access";
        private const string TARDIGRADE_BUCKET = "tardigrade-bucket";
        private const string TARDIGRADE_FOLDER = "tardigrade-folder";
        #endregion

        private const string STORJ_AUTH_METHOD = "storj-auth-method";
        private const string STORJ_SATELLITE = "storj-satellite";
        private const string STORJ_API_KEY = "storj-api-key";
        private const string STORJ_SECRET = "storj-secret";
        private const string STORJ_SHARED_ACCESS = "storj-shared-access";
        private const string STORJ_BUCKET = "storj-bucket";
        private const string STORJ_FOLDER = "storj-folder";

        private const string PROTOCOL_KEY = "tardigrade"; //Should be "storj" - but for historic reasons this stays "tardigrade" to not affect existing configurations
        private const string STORJ_PARTNER_ID = "duplicati";

        private readonly string _satellite;
        private readonly string _api_key;
        private readonly string _secret;
        private readonly string _bucket;
        private readonly string _folder;
        private Access _access;
        private IBucketService _bucketService;
        private IObjectService _objectService;

        public static readonly Dictionary<string, string> KNOWN_STORJ_SATELLITES = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){
            { "US Central", "us1.storj.io:7777" },
            { "Asia East", "ap1.storj.io:7777" },
            { "Europe", "eu1.storj.io:7777" },
        };

        public static readonly Dictionary<string, string> KNOWN_AUTHENTICATION_METHODS = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase){
            { "API key", "API key" },
            { "Access grant", "Access grant" },
        };

        [DllImport("kernel32.dll")]
        protected static extern IntPtr LoadLibrary(string filename);

        private static bool _libraryLoaded = false;
        private static void InitStorjLibrary()
        {
            if (_libraryLoaded)
                return;

            if (Duplicati.Library.Common.Platform.IsClientWindows) //We need to init only on Windows to distinguish between x64 and x86
            {
                if (System.Environment.Is64BitProcess)
                {
                    var res = LoadLibrary("win-x64/storj_uplink.dll");
                }
                else
                {
                    var res = LoadLibrary("win-x86/storj_uplink.dll");
                }
            }
            Access.SetTempDirectory(Library.Utility.TempFolder.SystemTempPath);
            _libraryLoaded = true;
        }

        // ReSharper disable once UnusedMember.Global
        // This constructor is needed by the BackendLoader.
        public Tardigrade()
        {
        }

        // ReSharper disable once UnusedMember.Global
        // This constructor is needed by the BackendLoader.
        public Tardigrade(string url, Dictionary<string, string> options)
        {
            InitStorjLibrary();

            options.TryGetValue(STORJ_AUTH_METHOD, out string auth_method);
            if (string.IsNullOrEmpty(auth_method))
                auth_method = options[TARDIGRADE_AUTH_METHOD];
            if (auth_method == "Access grant")
            {
                //Create an access from the access grant
                options.TryGetValue(STORJ_SHARED_ACCESS, out string shared_access);
                if (string.IsNullOrEmpty(shared_access))
                    shared_access = options[TARDIGRADE_SHARED_ACCESS];
                _access = new Access(shared_access, new Config() { UserAgent = STORJ_PARTNER_ID });
            }
            else
            {
                //Create an access for a satellite, API key and encryption passphrase
                options.TryGetValue(STORJ_SATELLITE, out _satellite);
                if (string.IsNullOrEmpty(_satellite))
                    _satellite = options[TARDIGRADE_SATELLITE];

                if (options.ContainsKey(TARDIGRADE_API_KEY) || options.ContainsKey(STORJ_API_KEY))
                {
                    options.TryGetValue(STORJ_API_KEY, out _api_key);
                    if (string.IsNullOrEmpty(_api_key))
                        _api_key = options[TARDIGRADE_API_KEY];
                }
                if (options.ContainsKey(TARDIGRADE_SECRET) || options.ContainsKey(STORJ_SECRET))
                {
                    options.TryGetValue(STORJ_SECRET, out _secret);
                    if (string.IsNullOrEmpty(_secret))
                        _secret = options[TARDIGRADE_SECRET];
                }

                _access = new Access(_satellite, _api_key, _secret, new Config() { UserAgent = STORJ_PARTNER_ID });
            }

            _bucketService = new BucketService(_access);
            _objectService = new ObjectService(_access);

            //If no bucket was provided use the default "duplicati"-bucket
            if (options.ContainsKey(TARDIGRADE_BUCKET) || options.ContainsKey(STORJ_BUCKET))
            {
                options.TryGetValue(STORJ_BUCKET, out _bucket);
                if (string.IsNullOrEmpty(_bucket))
                    _bucket = options[TARDIGRADE_BUCKET];
            }
            else
            {
                _bucket = "duplicati";
            }

            if (options.ContainsKey(TARDIGRADE_FOLDER) || options.ContainsKey(STORJ_FOLDER))
            {
                options.TryGetValue(STORJ_FOLDER, out _folder);
                if (string.IsNullOrEmpty(_folder))
                    _folder = options[TARDIGRADE_FOLDER];
            }
        }

        public string DisplayName
        {
            get { return Strings.Storj.DisplayName; }
        }

        public string ProtocolKey => PROTOCOL_KEY;

        public IList<ICommandLineArgument> SupportedCommands
        {
            get
            {
                return new List<ICommandLineArgument>(new ICommandLineArgument[] {
                    //Obsolete ones
                    new CommandLineArgument(TARDIGRADE_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, "API key", null, null, "This is deprecated, use storj-auth-method instead"),
                    new CommandLineArgument(TARDIGRADE_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, "us1.storj.io:7777", null, null, "This is deprecated, use storj-satellite instead"),
                    new CommandLineArgument(TARDIGRADE_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong, null, null, null, "This is deprecated, use storj-api-key instead"),
                    new CommandLineArgument(TARDIGRADE_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong, null, null, null, "This is deprecated, use storj-secret instead"),
                    new CommandLineArgument(TARDIGRADE_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong, null, null, null, "This is deprecated, use storj-shared-access instead"),
                    new CommandLineArgument(TARDIGRADE_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong, null, null, null, "This is deprecated, use storj-bucket instead"),
                    new CommandLineArgument(TARDIGRADE_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong, null, null, null, "This is deprecated, use storj-folder instead"),

                    //Current ones
                    new CommandLineArgument(STORJ_AUTH_METHOD, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAuthMethodDescriptionShort, Strings.Storj.StorjAuthMethodDescriptionLong, "API key"),
                    new CommandLineArgument(STORJ_SATELLITE, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSatelliteDescriptionShort, Strings.Storj.StorjSatelliteDescriptionLong, "us1.storj.io:7777"),
                    new CommandLineArgument(STORJ_API_KEY, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjAPIKeyDescriptionShort, Strings.Storj.StorjAPIKeyDescriptionLong),
                    new CommandLineArgument(STORJ_SECRET, CommandLineArgument.ArgumentType.Password, Strings.Storj.StorjSecretDescriptionShort, Strings.Storj.StorjSecretDescriptionLong),
                    new CommandLineArgument(STORJ_SHARED_ACCESS, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjSharedAccessDescriptionShort, Strings.Storj.StorjSharedAccessDescriptionLong),
                    new CommandLineArgument(STORJ_BUCKET, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjBucketDescriptionShort, Strings.Storj.StorjBucketDescriptionLong),
                    new CommandLineArgument(STORJ_FOLDER, CommandLineArgument.ArgumentType.String, Strings.Storj.StorjFolderDescriptionShort, Strings.Storj.StorjFolderDescriptionLong),
                });
            }
        }

        public string Description
        {
            get
            {
                return Strings.Storj.Description;
            }
        }

        public string[] DNSName
        {
            get
            {
                return new string[0];
            }
        }

        public void CreateFolder()
        {
            //Storj DCS has no folders
        }

        public void Delete(string remotename)
        {
            var deleteTask = DeleteAsync(remotename);
            deleteTask.Wait();
        }

        public async Task DeleteAsync(string remotename)
        {
            try
            {
                var bucket = await _bucketService.EnsureBucketAsync(_bucket);
                await _objectService.DeleteObjectAsync(bucket, GetBasePath() + remotename);
            }
            catch (Exception root)
            {
                throw new FileMissingException(root);
            }
        }

        public void Dispose()
        {
            if (_objectService != null)
            {
                _objectService = null;
            }
            if (_bucketService != null)
            {
                _bucketService = null;
            }
            if (_access != null)
            {
                _access.Dispose();
                _access = null;
            }
        }

        public void Get(string remotename, string filename)
        {
            var getTask = GetAsync(remotename, filename);
            getTask.Wait();
        }

        public async Task GetAsync(string remotename, string filename)
        {
            var bucket = await _bucketService.EnsureBucketAsync(_bucket);
            var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false);
            await download.StartDownloadAsync();

            if (download.Completed)
            {
                using (FileStream file = new FileStream(filename, FileMode.Create))
                {
                    await file.WriteAsync(download.DownloadedBytes, 0, (int)download.BytesReceived);
                    await file.FlushAsync().ConfigureAwait(false);
                }
            }
        }

        public void Get(string remotename, Stream stream)
        {
            var getTask = GetAsync(remotename, stream);
            getTask.Wait();
        }

        public async Task GetAsync(string remotename, Stream stream)
        {
            int index = 0;
            var bucket = await _bucketService.EnsureBucketAsync(_bucket);
            var download = await _objectService.DownloadObjectAsync(bucket, GetBasePath() + remotename, new DownloadOptions(), false);
            download.DownloadOperationProgressChanged += (op) =>
            {
                int newPartLength = (int)op.BytesReceived - index;
                byte[] newPart = new byte[newPartLength];
                Array.Copy(op.DownloadedBytes, index, newPart, 0, newPartLength);
                stream.Write(newPart, 0, newPartLength);
                index = index + newPartLength;
            };
            await download.StartDownloadAsync();
        }

        public IEnumerable<IFileEntry> List()
        {
            var listTask = ListAsync();
            listTask.Wait();
            return listTask.Result;
        }

        private async Task<IEnumerable<IFileEntry>> ListAsync()
        {
            List<StorjFile> files = new List<StorjFile>();
            var bucket = await _bucketService.EnsureBucketAsync(_bucket);
            var prefix = GetBasePath();
            var objects = await _objectService.ListObjectsAsync(bucket, new ListObjectsOptions { Recursive = true, System = true, Custom = true, Prefix = prefix });

            foreach (var obj in objects.Items)
            {
                StorjFile file = new StorjFile(obj);
                if (prefix != "")
                {
                    file.Name = file.Name.Replace(prefix, "");
                }
                files.Add(file);
            }

            return files;
        }

        public async Task PutAsync(string remotename, string filename, CancellationToken cancelToken)
        {
            using (FileStream fs = File.Open(filename, FileMode.Open, FileAccess.Read, FileShare.Read))
                await PutAsync(remotename, fs, cancelToken);
        }

        public async Task PutAsync(string remotename, Stream stream, CancellationToken cancelToken)
        {
            var bucket = await _bucketService.EnsureBucketAsync(_bucket);
            CustomMetadata custom = new CustomMetadata();
            custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_ACCESS, Value = DateTime.Now.ToUniversalTime().ToString("O") });
            custom.Entries.Add(new CustomMetadataEntry { Key = StorjFile.STORJ_LAST_MODIFICATION, Value = DateTime.Now.ToUniversalTime().ToString("O") });
            var upload = await _objectService.UploadObjectAsync(bucket, GetBasePath() + remotename, new UploadOptions(), stream, custom, false);
            await upload.StartUploadAsync();
        }

        public void Test()
        {
            var testTask = TestAsync();
            testTask.Wait(10000);
            if (!testTask.Result)
            {
                throw new Exception(Strings.Storj.TestConnectionFailed);
            }
        }

        /// <summary>
        /// Test the connection by:
        /// - creating the bucket (if it not already exists)
        /// - uploading 256 random bytes to a test-file
        /// - downloading the file back and expecting 256 bytes
        /// </summary>
        /// <returns>true, if the test was successfull or and exception</returns>
        private async Task<bool> TestAsync()
        {
            string testFileName = GetBasePath() + "duplicati_test.dat";

            var bucket = await _bucketService.EnsureBucketAsync(_bucket);
            var upload = await _objectService.UploadObjectAsync(bucket, testFileName, new UploadOptions(), GetRandomBytes(256), false);
            await upload.StartUploadAsync();

            var download = await _objectService.DownloadObjectAsync(bucket, testFileName, new DownloadOptions(), false);
            await download.StartDownloadAsync();

            await _objectService.DeleteObjectAsync(bucket, testFileName);

            if (download.Failed || download.BytesReceived != 256)
            {
                throw new Exception(download.ErrorMessage);
            }

            return true;
        }

        /// <summary>
        /// Gets the base path - depending on there is a folder set or not
        /// </summary>
        /// <returns>The base path within a bucket where the backup shall be placed</returns>
        private string GetBasePath()
        {
            if (!string.IsNullOrEmpty(_folder))
                return _folder + "/";
            else
                return "";
        }

        /// <summary>
        /// Creates some random bytes with the given length - just for testing the connection
        /// </summary>
        /// <param name="length">The length of the bytes to create</param>
        /// <returns>A byte-array with the given length</returns>
        private static byte[] GetRandomBytes(long length)
        {
            byte[] bytes = new byte[length];
            Random rand = new Random();
            rand.NextBytes(bytes);

            return bytes;
        }
    }
}