diff options
author | Grzegorz Bizon <grzegorz@gitlab.com> | 2017-06-08 08:29:35 +0300 |
---|---|---|
committer | Kamil TrzciĆski <ayufan@ayufan.eu> | 2018-02-28 22:00:27 +0300 |
commit | 52c3b8f31264230814d2ffa79d0987c1491676b3 (patch) | |
tree | d5827bc9bd891c1dd602eb3cdd4e4062d2e85589 /app/uploaders | |
parent | 64701b51aeacf4f4f932f205a2d831880b757a43 (diff) |
Merge branch 'zj-object-store-artifacts' into 'master'
Object store for artifacts
Closes gitlab-ce#29203
See merge request !1762
Diffstat (limited to 'app/uploaders')
-rw-r--r-- | app/uploaders/artifact_uploader.rb | 18 | ||||
-rw-r--r-- | app/uploaders/object_store_uploader.rb | 139 |
2 files changed, 147 insertions, 10 deletions
diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb index 14addb6cf14..0bd2bd4f422 100644 --- a/app/uploaders/artifact_uploader.rb +++ b/app/uploaders/artifact_uploader.rb @@ -1,7 +1,5 @@ -class ArtifactUploader < GitlabUploader - storage :file - - attr_reader :job, :field +class ArtifactUploader < ObjectStoreUploader + storage_options Gitlab.config.artifacts def self.local_artifacts_store Gitlab.config.artifacts.path @@ -11,12 +9,12 @@ class ArtifactUploader < GitlabUploader File.join(self.local_artifacts_store, 'tmp/uploads/') end - def initialize(job, field) - @job, @field = job, field - end - def store_dir - default_local_path + if file_storage? + default_local_path + else + default_path + end end def cache_dir @@ -34,6 +32,6 @@ class ArtifactUploader < GitlabUploader end def default_path - File.join(job.created_at.utc.strftime('%Y_%m'), job.project_id.to_s, job.id.to_s) + File.join(subject.created_at.utc.strftime('%Y_%m'), subject.project_id.to_s, subject.id.to_s) end end diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb new file mode 100644 index 00000000000..32d4d31b37c --- /dev/null +++ b/app/uploaders/object_store_uploader.rb @@ -0,0 +1,139 @@ +require 'fog/aws' +require 'carrierwave/storage/fog' + +class ObjectStoreUploader < GitlabUploader + before :store, :set_default_local_store + before :store, :verify_license! + + LOCAL_STORE = 1 + REMOTE_STORE = 2 + + class << self + def storage_options(options) + @storage_options = options + end + + def object_store_options + @storage_options&.object_store + end + + def object_store_enabled? + object_store_options&.enabled + end + end + + attr_reader :subject, :field + + def initialize(subject, field) + @subject = subject + @field = field + end + + def object_store + subject.public_send(:"#{field}_store") + end + + def object_store=(value) + @storage = nil + subject.public_send(:"#{field}_store=", value) + end + + def use_file + if file_storage? + return yield path + end + + begin + cache_stored_file! + yield cache_path + ensure + cache_storage.delete_dir!(cache_path(nil)) + end + end + + def filename + super || file&.filename + end + + def migrate!(new_store) + raise 'Undefined new store' unless new_store + + return unless object_store != new_store + return unless file + + old_file = file + old_store = object_store + + # for moving remote file we need to first store it locally + cache_stored_file! unless file_storage? + + # change storage + self.object_store = new_store + + storage.store!(file).tap do |new_file| + # since we change storage store the new storage + # in case of failure delete new file + begin + subject.save! + rescue => e + new_file.delete + self.object_store = old_store + raise e + end + + old_file.delete + end + end + + def fog_directory + self.class.object_store_options.remote_directory + end + + def fog_credentials + self.class.object_store_options.connection + end + + def fog_public + false + end + + def move_to_store + file.try(:storage) == storage + end + + def move_to_cache + file.try(:storage) == cache_storage + end + + # We block storing artifacts on Object Storage, not receiving + def verify_license!(new_file) + return if file_storage? + + raise 'Object Storage feature is missing' unless subject.project.feature_available?(:object_storage) + end + + private + + def set_default_local_store(new_file) + self.object_store = LOCAL_STORE unless self.object_store + end + + def storage + @storage ||= + if object_store == REMOTE_STORE + remote_storage + else + local_storage + end + end + + def remote_storage + raise 'Object Storage is not enabled' unless self.class.object_store_enabled? + + CarrierWave::Storage::Fog.new(self) + end + + def local_storage + CarrierWave::Storage::File.new(self) + end +end |