Add latest changes from gitlab-org/gitlab@master

4 files changed, 87 insertions, 27 deletions

diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index c5fb39b7b52..b30258123d4 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -10,11 +10,9 @@ module Gitlab
 
         StrategyError = Class.new(StandardError)
 
-        AFTER_EXPORT_LOCK_FILE_NAME = '.after_export_action'
-
         private
 
-        attr_reader :project, :current_user
+        attr_reader :project, :current_user, :lock_file
 
         public
 
@@ -29,8 +27,9 @@ module Gitlab
         def execute(current_user, project)
           @project = project
 
-          return unless @project.export_status == :finished
-
+          ensure_export_ready!
+          ensure_lock_files_path!
+          @lock_file = File.join(lock_files_path, SecureRandom.hex)
           @current_user = current_user
 
           if invalid?
@@ -48,19 +47,32 @@ module Gitlab
           false
         ensure
           delete_after_export_lock
+          delete_export_file
+          delete_archive_path
         end
 
         def to_json(options = {})
           @options.to_h.merge!(klass: self.class.name).to_json
         end
 
-        def self.lock_file_path(project)
-          return unless project.export_path || export_file_exists?
+        def ensure_export_ready!
+          raise StrategyError unless project.export_file_exists?
+        end
+
+        def ensure_lock_files_path!
+          FileUtils.mkdir_p(lock_files_path) unless Dir.exist?(lock_files_path)
+        end
+
+        def lock_files_path
+          project.import_export_shared.lock_files_path
+        end
 
-          lock_path = project.import_export_shared.archive_path
+        def archive_path
+          project.import_export_shared.archive_path
+        end
 
-          mkdir_p(lock_path)
-          File.join(lock_path, AFTER_EXPORT_LOCK_FILE_NAME)
+        def locks_present?
+          project.import_export_shared.locks_present?
         end
 
         protected
@@ -69,25 +81,33 @@ module Gitlab
           raise NotImplementedError
         end
 
+        def delete_export?
+          true
+        end
+
         private
 
+        def delete_export_file
+          return if locks_present? || !delete_export?
+
+          project.remove_exports
+        end
+
+        def delete_archive_path
+          FileUtils.rm_rf(archive_path) if File.directory?(archive_path)
+        end
+
         def create_or_update_after_export_lock
-          FileUtils.touch(self.class.lock_file_path(project))
+          FileUtils.touch(lock_file)
         end
 
         def delete_after_export_lock
-          lock_file = self.class.lock_file_path(project)
-
           FileUtils.rm(lock_file) if lock_file.present? && File.exist?(lock_file)
         end
 
         def log_validation_errors
           errors.full_messages.each { |msg| project.import_export_shared.add_error_message(msg) }
         end
-
-        def export_file_exists?
-          project.export_file_exists?
-        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
index 1b391314a74..39a6090ad87 100644
--- a/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/download_notification_strategy.rb
@@ -4,6 +4,12 @@ module Gitlab
   module ImportExport
     module AfterExportStrategies
       class DownloadNotificationStrategy < BaseAfterExportStrategy
+        protected
+
+        def delete_export?
+          false
+        end
+
         private
 
         def strategy_execute
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index aaa70f0b36d..fd98bc2caad 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -24,8 +24,6 @@ module Gitlab
 
         def strategy_execute
           handle_response_error(send_file)
-
-          project.remove_exports
         end
 
         def handle_response_error(response)
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 725c1101d70..9144a4406cd 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -1,10 +1,32 @@
 # frozen_string_literal: true
-
+#
+# This class encapsulates the directories used by project import/export:
+#
+# 1. The project export job first generates the project metadata tree
+#    (e.g. `project.json) and repository bundle (e.g. `project.bundle`)
+#    inside a temporary `export_path`
+#    (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA/:randomB).
+#
+# 2. The job then creates a tarball (e.g. `project.tar.gz`) in
+#    `archive_path` (e.g. /path/to/shared/tmp/project_exports/namespace/project/:randomA).
+#    CarrierWave moves this tarball files into its permanent location.
+#
+# 3. Lock files are used to indicate whether a project is in the
+#    `after_export` state.  These are stored in a directory
+#    (e.g. /path/to/shared/tmp/project_exports/namespace/project/locks. The
+#    number of lock files present signifies how many concurrent project
+#    exports are running. Note that this assumes the temporary directory
+#    is a shared mount:
+#    https://gitlab.com/gitlab-org/gitlab/issues/32203
+#
+# NOTE: Stale files should be cleaned up via ImportExportCleanupService.
 module Gitlab
   module ImportExport
     class Shared
       attr_reader :errors, :project
 
+      LOCKS_DIRECTORY = 'locks'
+
       def initialize(project)
         @project = project
         @errors = []
@@ -12,17 +34,27 @@ module Gitlab
       end
 
       def active_export_count
-        Dir[File.join(archive_path, '*')].count { |name| File.directory?(name) }
+        Dir[File.join(base_path, '*')].count { |name| File.basename(name) != LOCKS_DIRECTORY && File.directory?(name) }
       end
 
+      # The path where the project metadata and repository bundle is saved
       def export_path
         @export_path ||= Gitlab::ImportExport.export_path(relative_path: relative_path)
       end
 
+      # The path where the tarball is saved
       def archive_path
         @archive_path ||= Gitlab::ImportExport.export_path(relative_path: relative_archive_path)
       end
 
+      def base_path
+        @base_path ||= Gitlab::ImportExport.export_path(relative_path: relative_base_path)
+      end
+
+      def lock_files_path
+        @locks_files_path ||= File.join(base_path, LOCKS_DIRECTORY)
+      end
+
       def error(error)
         log_error(message: error.message, caller: caller[0].dup)
         log_debug(backtrace: error.backtrace&.join("\n"))
@@ -37,16 +69,24 @@ module Gitlab
       end
 
       def after_export_in_progress?
-        File.exist?(after_export_lock_file)
+        locks_present?
+      end
+
+      def locks_present?
+        Dir.exist?(lock_files_path) && !Dir.empty?(lock_files_path)
       end
 
       private
 
       def relative_path
-        File.join(relative_archive_path, SecureRandom.hex)
+        @relative_path ||= File.join(relative_archive_path, SecureRandom.hex)
       end
 
       def relative_archive_path
+        @relative_archive_path ||= File.join(@project.disk_path, SecureRandom.hex)
+      end
+
+      def relative_base_path
         @project.disk_path
       end
 
@@ -70,10 +110,6 @@ module Gitlab
       def filtered_error_message(message)
         Projects::ImportErrorFilter.filter_message(message)
       end
-
-      def after_export_lock_file
-        AfterExportStrategies::BaseAfterExportStrategy.lock_file_path(project)
-      end
     end
   end
 end