diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-08-20 21:42:06 +0300 |
commit | 6e4e1050d9dba2b7b2523fdd1768823ab85feef4 (patch) | |
tree | 78be5963ec075d80116a932011d695dd33910b4e /lib/object_storage | |
parent | 1ce776de4ae122aba3f349c02c17cebeaa8ecf07 (diff) |
Add latest changes from gitlab-org/gitlab@13-3-stable-ee
Diffstat (limited to 'lib/object_storage')
-rw-r--r-- | lib/object_storage/config.rb | 83 | ||||
-rw-r--r-- | lib/object_storage/direct_upload.rb | 59 |
2 files changed, 121 insertions, 21 deletions
diff --git a/lib/object_storage/config.rb b/lib/object_storage/config.rb new file mode 100644 index 00000000000..d0777914cb5 --- /dev/null +++ b/lib/object_storage/config.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +module ObjectStorage + class Config + attr_reader :options + + def initialize(options) + @options = options.to_hash.deep_symbolize_keys + end + + def credentials + @credentials ||= options[:connection] || {} + end + + def storage_options + @storage_options ||= options[:storage_options] || {} + end + + def enabled? + options[:enabled] + end + + def bucket + options[:remote_directory] + end + + def consolidated_settings? + options.fetch(:consolidated_settings, false) + end + + # AWS-specific options + def aws? + provider == 'AWS' + end + + def use_iam_profile? + Gitlab::Utils.to_boolean(credentials[:use_iam_profile], default: false) + end + + def use_path_style? + Gitlab::Utils.to_boolean(credentials[:path_style], default: false) + end + + def server_side_encryption + storage_options[:server_side_encryption] + end + + def server_side_encryption_kms_key_id + storage_options[:server_side_encryption_kms_key_id] + end + + def provider + credentials[:provider].to_s + end + # End AWS-specific options + + def google? + provider == 'Google' + end + + def azure? + provider == 'AzureRM' + end + + def fog_attributes + @fog_attributes ||= begin + return {} unless enabled? && aws? + return {} unless server_side_encryption.present? + + aws_server_side_encryption_headers.compact + end + end + + private + + def aws_server_side_encryption_headers + { + 'x-amz-server-side-encryption' => server_side_encryption, + 'x-amz-server-side-encryption-aws-kms-key-id' => server_side_encryption_kms_key_id + } + end + end +end diff --git a/lib/object_storage/direct_upload.rb b/lib/object_storage/direct_upload.rb index 76f92f62e9c..90199114f2c 100644 --- a/lib/object_storage/direct_upload.rb +++ b/lib/object_storage/direct_upload.rb @@ -22,20 +22,20 @@ module ObjectStorage MAXIMUM_MULTIPART_PARTS = 100 MINIMUM_MULTIPART_SIZE = 5.megabytes - attr_reader :credentials, :bucket_name, :object_name - attr_reader :has_length, :maximum_size, :consolidated_settings + attr_reader :config, :credentials, :bucket_name, :object_name + attr_reader :has_length, :maximum_size - def initialize(credentials, bucket_name, object_name, has_length:, maximum_size: nil, consolidated_settings: false) + def initialize(config, object_name, has_length:, maximum_size: nil) unless has_length raise ArgumentError, 'maximum_size has to be specified if length is unknown' unless maximum_size end - @credentials = credentials - @bucket_name = bucket_name + @config = config + @credentials = config.credentials + @bucket_name = config.bucket @object_name = object_name @has_length = has_length @maximum_size = maximum_size - @consolidated_settings = consolidated_settings end def to_hash @@ -62,8 +62,16 @@ module ObjectStorage end def workhorse_client_hash - return {} unless aws? + if config.aws? + workhorse_aws_hash + elsif config.azure? + workhorse_azure_hash + else + {} + end + end + def workhorse_aws_hash { UseWorkhorseClient: use_workhorse_s3_client?, RemoteTempObjectID: object_name, @@ -73,8 +81,25 @@ module ObjectStorage Bucket: bucket_name, Region: credentials[:region], Endpoint: credentials[:endpoint], - PathStyle: credentials.fetch(:path_style, false), - UseIamProfile: credentials.fetch(:use_iam_profile, false) + PathStyle: config.use_path_style?, + UseIamProfile: config.use_iam_profile?, + ServerSideEncryption: config.server_side_encryption, + SSEKMSKeyID: config.server_side_encryption_kms_key_id + }.compact + } + } + end + + def workhorse_azure_hash + { + # Azure requires Workhorse client because direct uploads can't + # use pre-signed URLs without buffering the whole file to disk. + UseWorkhorseClient: true, + RemoteTempObjectID: object_name, + ObjectStorage: { + Provider: 'AzureRM', + GoCloudConfig: { + URL: "azblob://#{bucket_name}" } } } @@ -82,7 +107,7 @@ module ObjectStorage def use_workhorse_s3_client? return false unless Feature.enabled?(:use_workhorse_s3_client, default_enabled: true) - return false unless credentials.fetch(:use_iam_profile, false) || consolidated_settings + return false unless config.use_iam_profile? || config.consolidated_settings? # The Golang AWS SDK does not support V2 signatures return false unless credentials.fetch(:aws_signature_version, 4).to_i >= 4 @@ -95,7 +120,7 @@ module ObjectStorage # Implements https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html def get_url - if google? + if config.google? connection.get_object_https_url(bucket_name, object_name, expire_at) else connection.get_object_url(bucket_name, object_name, expire_at) @@ -169,23 +194,15 @@ module ObjectStorage ].min end - def aws? - provider == 'AWS' - end - - def google? - provider == 'Google' - end - def requires_multipart_upload? - aws? && !has_length + config.aws? && !has_length end def upload_id return unless requires_multipart_upload? strong_memoize(:upload_id) do - new_upload = connection.initiate_multipart_upload(bucket_name, object_name) + new_upload = connection.initiate_multipart_upload(bucket_name, object_name, config.fog_attributes) new_upload.body["UploadId"] end end |