diff options
author | Douwe Maan <douwe@gitlab.com> | 2017-09-08 00:27:04 +0300 |
---|---|---|
committer | Kamil TrzciĆski <ayufan@ayufan.eu> | 2018-02-28 22:29:37 +0300 |
commit | bc76062774f01208403685965f4d780da4e03ebb (patch) | |
tree | e9e21e57b8783f25475648889372f4c3aed4eb3b /app/uploaders | |
parent | 5a69b51bc870f5b42ee3406ba77de02f44ef8d32 (diff) |
Merge branch 'jej/lfs-object-storage' into 'master'
Can migrate LFS objects to S3 style object storage
Closes #2841
See merge request !2760
Diffstat (limited to 'app/uploaders')
-rw-r--r-- | app/uploaders/artifact_uploader.rb | 24 | ||||
-rw-r--r-- | app/uploaders/lfs_object_uploader.rb | 21 | ||||
-rw-r--r-- | app/uploaders/object_store_uploader.rb | 60 |
3 files changed, 71 insertions, 34 deletions
diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb index 0bd2bd4f422..f6e32aae2fd 100644 --- a/app/uploaders/artifact_uploader.rb +++ b/app/uploaders/artifact_uploader.rb @@ -1,36 +1,16 @@ class ArtifactUploader < ObjectStoreUploader storage_options Gitlab.config.artifacts - def self.local_artifacts_store + def self.local_store_path Gitlab.config.artifacts.path end def self.artifacts_upload_path - File.join(self.local_artifacts_store, 'tmp/uploads/') - end - - def store_dir - if file_storage? - default_local_path - else - default_path - end - end - - def cache_dir - File.join(self.class.local_artifacts_store, 'tmp/cache') - end - - def work_dir - File.join(self.class.local_artifacts_store, 'tmp/work') + File.join(self.local_store_path, 'tmp/uploads/') end private - def default_local_path - File.join(self.class.local_artifacts_store, default_path) - end - def default_path File.join(subject.created_at.utc.strftime('%Y_%m'), subject.project_id.to_s, subject.id.to_s) end diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb index d11ebf0f9ca..8a5f599c1d3 100644 --- a/app/uploaders/lfs_object_uploader.rb +++ b/app/uploaders/lfs_object_uploader.rb @@ -1,19 +1,18 @@ -class LfsObjectUploader < GitlabUploader - storage :file +class LfsObjectUploader < ObjectStoreUploader + storage_options Gitlab.config.lfs + after :store, :schedule_migration_to_object_storage - def store_dir - "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}" - end - - def cache_dir - "#{Gitlab.config.lfs.storage_path}/tmp/cache" + def self.local_store_path + Gitlab.config.lfs.storage_path end def filename - model.oid[4..-1] + subject.oid[4..-1] end - def work_dir - File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work') + private + + def default_path + "#{subject.oid[0, 2]}/#{subject.oid[2, 2]}" end end diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb index 32d4d31b37c..3a742d4f715 100644 --- a/app/uploaders/object_store_uploader.rb +++ b/app/uploaders/object_store_uploader.rb @@ -20,6 +20,22 @@ class ObjectStoreUploader < GitlabUploader def object_store_enabled? object_store_options&.enabled end + + def background_upload_enabled? + object_store_options&.background_upload + end + + def object_store_credentials + @object_store_credentials ||= object_store_options&.connection&.to_hash&.deep_symbolize_keys + end + + def object_store_directory + object_store_options&.remote_directory + end + + def local_store_path + raise NotImplementedError + end end attr_reader :subject, :field @@ -38,6 +54,14 @@ class ObjectStoreUploader < GitlabUploader subject.public_send(:"#{field}_store=", value) end + def store_dir + if file_storage? + default_local_path + else + default_path + end + end + def use_file if file_storage? return yield path @@ -85,6 +109,12 @@ class ObjectStoreUploader < GitlabUploader end end + def schedule_migration_to_object_storage(new_file) + if self.class.object_store_enabled? && licensed? && file_storage? + ObjectStorageUploadWorker.perform_async(self.class.name, subject.class.name, field, subject.id) + end + end + def fog_directory self.class.object_store_options.remote_directory end @@ -109,7 +139,27 @@ class ObjectStoreUploader < GitlabUploader def verify_license!(new_file) return if file_storage? - raise 'Object Storage feature is missing' unless subject.project.feature_available?(:object_storage) + raise 'Object Storage feature is missing' unless licensed? + end + + def exists? + file.try(:exists?) + end + + def cache_dir + File.join(self.class.local_store_path, 'tmp/cache') + end + + # Override this if you don't want to save local files by default to the Rails.root directory + def work_dir + # Default path set by CarrierWave: + # https://github.com/carrierwaveuploader/carrierwave/blob/v1.1.0/lib/carrierwave/uploader/cache.rb#L182 + # CarrierWave.tmp_path + File.join(self.class.local_store_path, 'tmp/work') + end + + def licensed? + License.feature_available?(:object_storage) end private @@ -118,6 +168,14 @@ class ObjectStoreUploader < GitlabUploader self.object_store = LOCAL_STORE unless self.object_store end + def default_local_path + File.join(self.class.local_store_path, default_path) + end + + def default_path + raise NotImplementedError + end + def storage @storage ||= if object_store == REMOTE_STORE |