Merge branch 'jej/lfs-object-storage' into 'master'

Can migrate LFS objects to S3 style object storage

Closes #2841

See merge request !2760

3 files changed, 71 insertions, 34 deletions

diff --git a/app/uploaders/artifact_uploader.rb b/app/uploaders/artifact_uploader.rb
index 0bd2bd4f422..f6e32aae2fd 100644
--- a/app/uploaders/artifact_uploader.rb
+++ b/app/uploaders/artifact_uploader.rb
@@ -1,36 +1,16 @@
 class ArtifactUploader < ObjectStoreUploader
   storage_options Gitlab.config.artifacts
 
-  def self.local_artifacts_store
+  def self.local_store_path
     Gitlab.config.artifacts.path
   end
 
   def self.artifacts_upload_path
-    File.join(self.local_artifacts_store, 'tmp/uploads/')
-  end
-
-  def store_dir
-    if file_storage?
-      default_local_path
-    else
-      default_path
-    end
-  end
-
-  def cache_dir
-    File.join(self.class.local_artifacts_store, 'tmp/cache')
-  end
-
-  def work_dir
-    File.join(self.class.local_artifacts_store, 'tmp/work')
+    File.join(self.local_store_path, 'tmp/uploads/')
   end
 
   private
 
-  def default_local_path
-    File.join(self.class.local_artifacts_store, default_path)
-  end
-
   def default_path
     File.join(subject.created_at.utc.strftime('%Y_%m'), subject.project_id.to_s, subject.id.to_s)
   end
diff --git a/app/uploaders/lfs_object_uploader.rb b/app/uploaders/lfs_object_uploader.rb
index d11ebf0f9ca..8a5f599c1d3 100644
--- a/app/uploaders/lfs_object_uploader.rb
+++ b/app/uploaders/lfs_object_uploader.rb
@@ -1,19 +1,18 @@
-class LfsObjectUploader < GitlabUploader
-  storage :file
+class LfsObjectUploader < ObjectStoreUploader
+  storage_options Gitlab.config.lfs
+  after :store, :schedule_migration_to_object_storage
 
-  def store_dir
-    "#{Gitlab.config.lfs.storage_path}/#{model.oid[0, 2]}/#{model.oid[2, 2]}"
-  end
-
-  def cache_dir
-    "#{Gitlab.config.lfs.storage_path}/tmp/cache"
+  def self.local_store_path
+    Gitlab.config.lfs.storage_path
   end
 
   def filename
-    model.oid[4..-1]
+    subject.oid[4..-1]
   end
 
-  def work_dir
-    File.join(Gitlab.config.lfs.storage_path, 'tmp', 'work')
+  private
+
+  def default_path
+    "#{subject.oid[0, 2]}/#{subject.oid[2, 2]}"
   end
 end
diff --git a/app/uploaders/object_store_uploader.rb b/app/uploaders/object_store_uploader.rb
index 32d4d31b37c..3a742d4f715 100644
--- a/app/uploaders/object_store_uploader.rb
+++ b/app/uploaders/object_store_uploader.rb
@@ -20,6 +20,22 @@ class ObjectStoreUploader < GitlabUploader
     def object_store_enabled?
       object_store_options&.enabled
     end
+
+    def background_upload_enabled?
+      object_store_options&.background_upload
+    end
+
+    def object_store_credentials
+      @object_store_credentials ||= object_store_options&.connection&.to_hash&.deep_symbolize_keys
+    end
+
+    def object_store_directory
+      object_store_options&.remote_directory
+    end
+
+    def local_store_path
+      raise NotImplementedError
+    end
   end
 
   attr_reader :subject, :field
@@ -38,6 +54,14 @@ class ObjectStoreUploader < GitlabUploader
     subject.public_send(:"#{field}_store=", value)
   end
 
+  def store_dir
+    if file_storage?
+      default_local_path
+    else
+      default_path
+    end
+  end
+
   def use_file
     if file_storage?
       return yield path
@@ -85,6 +109,12 @@ class ObjectStoreUploader < GitlabUploader
     end
   end
 
+  def schedule_migration_to_object_storage(new_file)
+    if self.class.object_store_enabled? && licensed? && file_storage?
+      ObjectStorageUploadWorker.perform_async(self.class.name, subject.class.name, field, subject.id)
+    end
+  end
+
   def fog_directory
     self.class.object_store_options.remote_directory
   end
@@ -109,7 +139,27 @@ class ObjectStoreUploader < GitlabUploader
   def verify_license!(new_file)
     return if file_storage?
 
-    raise 'Object Storage feature is missing' unless subject.project.feature_available?(:object_storage)
+    raise 'Object Storage feature is missing' unless licensed?
+  end
+
+  def exists?
+    file.try(:exists?)
+  end
+
+  def cache_dir
+    File.join(self.class.local_store_path, 'tmp/cache')
+  end
+
+  # Override this if you don't want to save local files by default to the Rails.root directory
+  def work_dir
+    # Default path set by CarrierWave:
+    # https://github.com/carrierwaveuploader/carrierwave/blob/v1.1.0/lib/carrierwave/uploader/cache.rb#L182
+    # CarrierWave.tmp_path
+    File.join(self.class.local_store_path, 'tmp/work')
+  end
+
+  def licensed?
+    License.feature_available?(:object_storage)
   end
 
   private
@@ -118,6 +168,14 @@ class ObjectStoreUploader < GitlabUploader
     self.object_store = LOCAL_STORE unless self.object_store
   end
 
+  def default_local_path
+    File.join(self.class.local_store_path, default_path)
+  end
+
+  def default_path
+    raise NotImplementedError
+  end
+
   def storage
     @storage ||=
       if object_store == REMOTE_STORE