Merge branch '48778-remove-old-storage-logic-from-import-export' into 'master'

Resolve "Remove old storage logic from Import/Export"

Closes #51144, #50109, #48778, and #51079

See merge request gitlab-org/gitlab-ce!21409

14 files changed, 26 insertions, 101 deletions

diff --git a/lib/api/project_export.rb b/lib/api/project_export.rb
index 15c57a2fc02..8562ae6d737 100644
--- a/lib/api/project_export.rb
+++ b/lib/api/project_export.rb
@@ -21,12 +21,8 @@ module API
         detail 'This feature was introduced in GitLab 10.6.'
       end
       get ':id/export/download' do
-        path = user_project.export_project_path
-
-        if path
-          present_disk_file!(path, File.basename(path), 'application/gzip')
-        elsif user_project.export_project_object_exists?
-          present_carrierwave_file!(user_project.import_export_upload.export_file)
+        if user_project.export_file_exists?
+          present_carrierwave_file!(user_project.export_file)
         else
           render_api_error!('404 Not found or has expired', 404)
         end
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index be3710c5b7f..53fe2f8e436 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -40,10 +40,6 @@ module Gitlab
       "#{basename[0..FILENAME_LIMIT]}_export.tar.gz"
     end
 
-    def object_storage?
-      Feature.enabled?(:import_export_object_storage)
-    end
-
     def version
       VERSION
     end
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index 83134bb0769..7cbf653dd97 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -53,7 +53,7 @@ module Gitlab
         end
 
         def self.lock_file_path(project)
-          return unless project.export_path || object_storage?
+          return unless project.export_path || export_file_exists?
 
           lock_path = project.import_export_shared.archive_path
 
@@ -83,8 +83,8 @@ module Gitlab
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
 
-        def object_storage?
-          project.export_project_object_exists?
+        def export_file_exists?
+          project.export_file_exists?
         end
       end
     end
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index dce8f89c0ab..4f29bdcea2c 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -23,7 +23,7 @@ module Gitlab
         def strategy_execute
           handle_response_error(send_file)
 
-          project.remove_exported_project_file
+          project.remove_exports
         end
 
         def handle_response_error(response)
@@ -40,15 +40,11 @@ module Gitlab
         def send_file
           Gitlab::HTTP.public_send(http_method.downcase, url, send_file_options) # rubocop:disable GitlabSecurity/PublicSend
         ensure
-          export_file.close if export_file && !object_storage?
+          export_file.close if export_file
         end
 
         def export_file
-          if object_storage?
-            project.import_export_upload.export_file.file.open
-          else
-            File.open(project.export_project_path)
-          end
+          project.export_file.open
         end
 
         def send_file_options
@@ -63,11 +59,7 @@ module Gitlab
         end
 
         def export_size
-          if object_storage?
-            project.import_export_upload.export_file.file.size
-          else
-            File.size(project.export_project_path)
-          end
+          project.export_file.file.size
         end
       end
     end
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index cfa595629f4..ded05f73cf8 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -19,7 +19,7 @@ module Gitlab
       private
 
       def avatar_export_file
-        @avatar_export_file ||= Dir["#{avatar_export_path}/*"].first
+        @avatar_export_file ||= Dir["#{avatar_export_path}/**/*"].first
       end
 
       def avatar_export_path
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb
index 31ef0490cb3..6ffebf83dd2 100644
--- a/lib/gitlab/import_export/avatar_saver.rb
+++ b/lib/gitlab/import_export/avatar_saver.rb
@@ -1,8 +1,6 @@
 module Gitlab
   module ImportExport
     class AvatarSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
@@ -14,19 +12,12 @@ module Gitlab
         Gitlab::ImportExport::UploadsManager.new(
           project: @project,
           shared: @shared,
-          relative_export_path: 'avatar',
-          from: avatar_path
+          relative_export_path: 'avatar'
         ).save
       rescue => e
         @shared.error(e)
         false
       end
-
-      private
-
-      def avatar_path
-        @project.avatar.path
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 4e179f63d8c..72d5b9b830c 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -92,8 +92,6 @@ module Gitlab
       end
 
       def remove_import_file
-        return unless Gitlab::ImportExport.object_storage?
-
         upload = @project.import_export_upload
 
         return unless upload&.import_file&.file
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index f4106e03a57..00ea4b833e2 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -199,7 +199,7 @@ module Gitlab
       end
 
       def excluded_keys_for_relation(relation)
-        @reader.attributes_finder.find_excluded_keys(relation)
+        reader.attributes_finder.find_excluded_keys(relation)
       end
     end
   end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index 3cd153a4fd2..59a74083395 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -18,7 +18,7 @@ module Gitlab
 
           Rails.logger.info("Saved project export #{archive_file}")
 
-          save_on_object_storage if use_object_storage?
+          save_upload
         else
           @shared.error(Gitlab::ImportExport::Error.new(error_message))
           false
@@ -27,10 +27,8 @@ module Gitlab
         @shared.error(e)
         false
       ensure
-        if use_object_storage?
-          remove_archive
-          remove_export_path
-        end
+        remove_archive
+        remove_export_path
       end
 
       private
@@ -51,7 +49,7 @@ module Gitlab
         @archive_file ||= File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(project: @project))
       end
 
-      def save_on_object_storage
+      def save_upload
         upload = ImportExportUpload.find_or_initialize_by(project: @project)
 
         File.open(archive_file) { |file| upload.export_file = file }
@@ -59,12 +57,8 @@ module Gitlab
         upload.save!
       end
 
-      def use_object_storage?
-        Gitlab::ImportExport.object_storage?
-      end
-
       def error_message
-        "Unable to save #{archive_file} into #{@shared.export_path}. Object storage enabled: #{use_object_storage?}"
+        "Unable to save #{archive_file} into #{@shared.export_path}."
       end
     end
   end
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index e0d4235e65b..8511319cb1c 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -5,18 +5,13 @@ module Gitlab
 
       UPLOADS_BATCH_SIZE = 100
 
-      def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
+      def initialize(project:, shared:, relative_export_path: 'uploads')
         @project = project
         @shared = shared
         @relative_export_path = relative_export_path
-        @from = from || default_uploads_path
       end
 
       def save
-        if File.file?(@from) && @relative_export_path == 'avatar'
-          copy_files(@from, File.join(uploads_export_path, @project.avatar.filename))
-        end
-
         copy_project_uploads
 
         true
@@ -55,17 +50,11 @@ module Gitlab
 
             copy_files(uploader.absolute_path, File.join(uploads_export_path, uploader.upload.path))
           else
-            next unless Gitlab::ImportExport.object_storage?
-
             download_and_copy(uploader)
           end
         end
       end
 
-      def default_uploads_path
-        FileUploader.absolute_base_dir(@project)
-      end
-
       def uploads_export_path
         @uploads_export_path ||= File.join(@shared.export_path, @relative_export_path)
       end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
index 25f85936227..b4313ff4cb4 100644
--- a/lib/gitlab/import_export/uploads_restorer.rb
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -2,30 +2,14 @@ module Gitlab
   module ImportExport
     class UploadsRestorer < UploadsSaver
       def restore
-        if Gitlab::ImportExport.object_storage?
-          Gitlab::ImportExport::UploadsManager.new(
-            project: @project,
-            shared: @shared
-          ).restore
-        elsif File.directory?(uploads_export_path)
-          copy_files(uploads_export_path, uploads_path)
-
-          true
-        else
-          true # Proceed without uploads
-        end
+        Gitlab::ImportExport::UploadsManager.new(
+          project: @project,
+          shared: @shared
+        ).restore
       rescue => e
         @shared.error(e)
         false
       end
-
-      def uploads_path
-        FileUploader.absolute_base_dir(@project)
-      end
-
-      def uploads_export_path
-        @uploads_export_path ||= File.join(@shared.export_path, 'uploads')
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index b3f17af5661..0275f686c5e 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -1,8 +1,6 @@
 module Gitlab
   module ImportExport
     class UploadsSaver
-      include Gitlab::ImportExport::CommandLineUtil
-
       def initialize(project:, shared:)
         @project = project
         @shared = shared
diff --git a/lib/gitlab/template_helper.rb b/lib/gitlab/template_helper.rb
index f24a01e6cf5..fc498dde723 100644
--- a/lib/gitlab/template_helper.rb
+++ b/lib/gitlab/template_helper.rb
@@ -1,24 +1,9 @@
 module Gitlab
   module TemplateHelper
-    include Gitlab::Utils::StrongMemoize
-
     def prepare_template_environment(file)
       return unless file
 
-      if Gitlab::ImportExport.object_storage?
-        params[:import_export_upload] = ImportExportUpload.new(import_file: file)
-      else
-        FileUtils.mkdir_p(File.dirname(import_upload_path))
-        FileUtils.copy_entry(file.path, import_upload_path)
-
-        params[:import_source] = import_upload_path
-      end
-    end
-
-    def import_upload_path
-      strong_memoize(:import_upload_path) do
-        Gitlab::ImportExport.import_upload_path(filename: tmp_filename)
-      end
+      params[:import_export_upload] = ImportExportUpload.new(import_file: file)
     end
 
     def tmp_filename
diff --git a/lib/tasks/gitlab/update_templates.rake b/lib/tasks/gitlab/update_templates.rake
index a25f7ce59c7..ef6a32d6730 100644
--- a/lib/tasks/gitlab/update_templates.rake
+++ b/lib/tasks/gitlab/update_templates.rake
@@ -6,6 +6,8 @@ namespace :gitlab do
 
   desc "GitLab | Update project templates"
   task :update_project_templates do
+    include Gitlab::ImportExport::CommandLineUtil
+
     if Rails.env.production?
       puts "This rake task is not meant fo production instances".red
       exit(1)
@@ -52,7 +54,7 @@ namespace :gitlab do
       end
 
       Projects::ImportExport::ExportService.new(project, admin).execute
-      FileUtils.cp(project.export_project_path, template.archive_path)
+      download_or_copy_upload(project.export_file, template.archive_path)
       Projects::DestroyService.new(admin, project).execute
       puts "Exported #{template.name}".green
     end