// Copyright (C) 2015, The Duplicati Team // http://www.duplicati.com, info@duplicati.com // // This library is free software; you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as // published by the Free Software Foundation; either version 2.1 of the // License, or (at your option) any later version. // // This library is distributed in the hope that it will be useful, but // WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU // Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public // License along with this library; if not, write to the Free Software // Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA using System; using System.Collections.Generic; using Duplicati.Library.Interface; using System.Linq; using Duplicati.Library.Utility; using Newtonsoft.Json; using Duplicati.Library.Strings; using System.Net; using System.Text; namespace Duplicati.Library.Backend.OpenStack { public class OpenStackStorage : IBackend, IStreamingBackend { private const string USERNAME_OPTION = "auth-username"; private const string PASSWORD_OPTION = "auth-password"; private const string TENANTNAME_OPTION = "openstack-tenant-name"; private const string AUTHURI_OPTION = "openstack-authuri"; private const string APIKEY_OPTION = "openstack-apikey"; private const string REGION_OPTION = "openstack-region"; private const int PAGE_LIMIT = 500; private string m_container; private string m_prefix; private string m_username; private string m_password; private string m_authUri; private string m_tenantName; private string m_apikey; private string m_region; protected string m_simplestorageendpoint; private WebHelper m_helper = null; private OpenStackAuthResponse.TokenClass m_accessToken; public static readonly KeyValuePair[] KNOWN_OPENSTACK_PROVIDERS = new KeyValuePair[] { new KeyValuePair("Rackspace US", "https://identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("Rackspace UK", "https://lon.identity.api.rackspacecloud.com/v2.0"), new KeyValuePair("OVH Cloud Storage", "https://auth.cloud.ovh.net/v2.0"), new KeyValuePair("Selectel Cloud Storage", "https://auth.selcdn.ru"), }; private class OpenStackAuthRequest { public class AuthContainer { [JsonProperty("RAX-KSKEY:apiKeyCredentials", NullValueHandling = NullValueHandling.Ignore)] public ApiKeyBasedRequest ApiCredentials { get; set; } [JsonProperty("passwordCredentials", NullValueHandling = NullValueHandling.Ignore)] public PasswordBasedRequest PasswordCredentials { get; set; } [JsonProperty("tenantName", NullValueHandling = NullValueHandling.Ignore)] public string TenantName { get; set; } [JsonProperty("token", NullValueHandling = NullValueHandling.Ignore)] public TokenBasedRequest Token { get; set; } } public class ApiKeyBasedRequest { public string username { get; set; } public string apiKey { get; set; } } public class PasswordBasedRequest { public string username { get; set; } public string password { get; set; } public string tenantName { get; set; } } public class TokenBasedRequest { public string id { get; set; } } public AuthContainer auth { get; set; } public OpenStackAuthRequest(string tenantname, string username, string password, string apikey) { this.auth = new AuthContainer(); this.auth.TenantName = tenantname; if (string.IsNullOrEmpty(apikey)) { this.auth.PasswordCredentials = new PasswordBasedRequest() { username = username, password = password, }; } else { this.auth.ApiCredentials = new ApiKeyBasedRequest() { username = username, apiKey = apikey }; } } } private class OpenStackAuthResponse { public AccessClass access { get; set; } public class TokenClass { public string id { get; set; } public DateTime? expires { get; set; } } public class EndpointItem { public string region { get; set; } public string tenantId { get; set; } public string publicURL { get; set; } public string internalURL { get; set; } } public class ServiceItem { public string name { get; set; } public string type { get; set; } public EndpointItem[] endpoints { get; set; } } public class AccessClass { public TokenClass token { get; set; } public ServiceItem[] serviceCatalog { get; set; } } } private class OpenStackStorageItem { public string name { get; set; } public DateTime? last_modified { get; set; } public long? bytes { get; set; } public string content_type { get; set; } public string subdir { get; set; } } private class WebHelper : JSONWebHelper { private OpenStackStorage m_parent; public WebHelper(OpenStackStorage parent) { m_parent = parent; } public override HttpWebRequest CreateRequest(string url, string method = null) { var req = base.CreateRequest(url, method); req.Headers["X-Auth-Token"] = m_parent.AccessToken; return req; } } public OpenStackStorage() { } public OpenStackStorage(string url, Dictionary options) { var uri = new Utility.Uri(url); m_container = uri.Host; m_prefix = "/" + uri.Path; if (!m_prefix.EndsWith("/")) m_prefix += "/"; // For OpenStack we do not use a leading slash if (m_prefix.StartsWith("/")) m_prefix = m_prefix.Substring(1); options.TryGetValue(USERNAME_OPTION, out m_username); options.TryGetValue(PASSWORD_OPTION, out m_password); options.TryGetValue(TENANTNAME_OPTION, out m_tenantName); options.TryGetValue(AUTHURI_OPTION, out m_authUri); options.TryGetValue(APIKEY_OPTION, out m_apikey); options.TryGetValue(REGION_OPTION, out m_region); if (string.IsNullOrWhiteSpace(m_username)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(USERNAME_OPTION)); if (string.IsNullOrWhiteSpace(m_authUri)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(AUTHURI_OPTION)); if (string.IsNullOrWhiteSpace(m_apikey)) { if (string.IsNullOrWhiteSpace(m_password)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(PASSWORD_OPTION)); if (string.IsNullOrWhiteSpace(m_tenantName)) throw new UserInformationException(Strings.OpenStack.MissingOptionError(TENANTNAME_OPTION)); } m_helper = new WebHelper(this); } protected virtual string AccessToken { get { if (m_accessToken == null || (m_accessToken.expires.HasValue && (m_accessToken.expires.Value - DateTime.UtcNow).TotalSeconds < 30)) GetAuthResponse(); return m_accessToken.id; } } private string JoinUrls(string uri, string fragment) { fragment = fragment ?? ""; return uri + (uri.EndsWith("/") ? "" : "/") + (fragment.StartsWith("/") ? fragment.Substring(1) : fragment); } private string JoinUrls(string uri, string fragment1, string fragment2) { return JoinUrls(JoinUrls(uri, fragment1), fragment2); } private OpenStackAuthResponse GetAuthResponse() { var helper = new JSONWebHelper(); var req = helper.CreateRequest(JoinUrls(m_authUri, "tokens")); req.Accept = "application/json"; req.Method = "POST"; var resp = helper.ReadJSONResponse( req, new OpenStackAuthRequest(m_tenantName, m_username, m_password, m_apikey) ); m_accessToken = resp.access.token; // Grab the endpoint now that we have received it anyway var fileservice = resp.access.serviceCatalog.Where(x => string.Equals(x.type, "object-store", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); if (fileservice == null) throw new Exception("No object-store service found, is this service supported by the provider?"); var endpoint = fileservice.endpoints.Where(x => string.Equals(m_region, x.region)).FirstOrDefault() ?? fileservice.endpoints.First(); m_simplestorageendpoint = endpoint.publicURL; return resp; } protected virtual string SimpleStorageEndPoint { get { if (m_simplestorageendpoint == null) GetAuthResponse(); return m_simplestorageendpoint; } } #region IStreamingBackend implementation public void Put(string remotename, System.IO.Stream stream) { var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); using(m_helper.GetResponse(url, stream, "PUT")) { } } public void Get(string remotename, System.IO.Stream stream) { var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); try { using(var resp = m_helper.GetResponse(url)) using(var rs = AsyncHttpRequest.TrySetTimeout(resp.GetResponseStream())) Library.Utility.Utility.CopyStream(rs, stream); } catch(WebException wex) { if (wex.Response is HttpWebResponse && ((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound) throw new FileMissingException(); else throw; } } #endregion #region IBackend implementation public List List() { var res = new List(); var plainurl = JoinUrls(SimpleStorageEndPoint, m_container) + string.Format("?format=json&delimiter=/&limit={0}", PAGE_LIMIT); if (!string.IsNullOrEmpty(m_prefix)) plainurl += "&prefix=" + Library.Utility.Uri.UrlEncode(m_prefix); var url = plainurl; try { while(true) { var req = m_helper.CreateRequest(url); req.Accept = "application/json"; var items = m_helper.ReadJSONResponse(req); foreach(var n in items) { var name = n.name; if (name.StartsWith(m_prefix)) name = name.Substring(m_prefix.Length); if (n.bytes == null) res.Add(new FileEntry(name)); else if (n.last_modified == null) res.Add(new FileEntry(name, n.bytes.Value)); else res.Add(new FileEntry(name, n.bytes.Value, n.last_modified.Value, n.last_modified.Value)); } if (items.Length != PAGE_LIMIT) return res; // Prepare next listing entry url = plainurl + string.Format("&marker={0}", Library.Utility.Uri.UrlEncode(items.Last().name)); } } catch(WebException wex) { if (wex.Response is HttpWebResponse && (((HttpWebResponse)wex.Response).StatusCode == HttpStatusCode.NotFound)) throw new FolderMissingException(); else throw; } } public void Put(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.OpenRead(filename)) Put(remotename, fs); } public void Get(string remotename, string filename) { using (System.IO.FileStream fs = System.IO.File.Create(filename)) Get(remotename, fs); } public void Delete(string remotename) { var url = JoinUrls(SimpleStorageEndPoint, m_container, Library.Utility.Uri.UrlPathEncode(m_prefix + remotename)); m_helper.ReadJSONResponse(url, null, "DELETE"); } public void Test() { List(); } public void CreateFolder() { var url = JoinUrls(SimpleStorageEndPoint, m_container); using(m_helper.GetResponse(url, null, "PUT")) { } } public string DisplayName { get { return Strings.OpenStack.DisplayName; } } public string ProtocolKey { get { return "openstack"; } } public IList SupportedCommands { get { var authuris = new StringBuilder(); foreach(var s in KNOWN_OPENSTACK_PROVIDERS) authuris.AppendLine(string.Format("{0}: {1}", s.Key, s.Value)); return new List(new CommandLineArgument[] { new CommandLineArgument(USERNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.UsernameOptionShort, Strings.OpenStack.UsernameOptionLong), new CommandLineArgument(PASSWORD_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.PasswordOptionShort, Strings.OpenStack.PasswordOptionLong(TENANTNAME_OPTION)), new CommandLineArgument(TENANTNAME_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.TenantnameOptionShort, Strings.OpenStack.TenantnameOptionLong), new CommandLineArgument(APIKEY_OPTION, CommandLineArgument.ArgumentType.Password, Strings.OpenStack.ApikeyOptionShort, Strings.OpenStack.ApikeyOptionLong), new CommandLineArgument(AUTHURI_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.AuthuriOptionShort, Strings.OpenStack.AuthuriOptionLong(authuris.ToString())), new CommandLineArgument(REGION_OPTION, CommandLineArgument.ArgumentType.String, Strings.OpenStack.RegionOptionShort, Strings.OpenStack.RegionOptionLong), }); } } public string Description { get { return Strings.OpenStack.Description; } } #endregion #region IDisposable implementation public void Dispose() { } #endregion } }