Add latest changes from gitlab-org/gitlab@master

6 files changed, 203 insertions, 14 deletions

diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
index 454dc778b6b..f11b7a0a298 100644
--- a/lib/gitlab/import_export/error.rb
+++ b/lib/gitlab/import_export/error.rb
@@ -2,6 +2,13 @@
 
 module Gitlab
   module ImportExport
-    Error = Class.new(StandardError)
+    class Error < StandardError
+      def self.permission_error(user, importable)
+        self.new(
+          "User with ID: %s does not have required permissions for %s: %s with ID: %s" %
+          [user.id, importable.class.name, importable.name, importable.id]
+        )
+      end
+    end
   end
 end
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index e6f49dcac7a..cbaa6929efa 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -49,11 +49,7 @@ module Gitlab
           json = IO.read(@path)
           ActiveSupport::JSON.decode(json)
         rescue => e
-          @shared.logger.error(
-            group_id:   @group.id,
-            group_name: @group.name,
-            message:    "Import/Export error: #{e.message}"
-          )
+          @shared.error(e)
 
           raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
         end
diff --git a/lib/gitlab/import_export/project/base_task.rb b/lib/gitlab/import_export/project/base_task.rb
new file mode 100644
index 00000000000..6a7b24421c9
--- /dev/null
+++ b/lib/gitlab/import_export/project/base_task.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class BaseTask
+        include Gitlab::WithRequestStore
+
+        def initialize(opts, logger: Logger.new($stdout))
+          @project_path = opts.fetch(:project_path)
+          @file_path    = opts.fetch(:file_path)
+          @namespace    = Namespace.find_by_full_path(opts.fetch(:namespace_path))
+          @current_user = User.find_by_username(opts.fetch(:username))
+          @measurement_enabled = opts.fetch(:measurement_enabled)
+          @measurement = Gitlab::Utils::Measuring.new(logger: logger) if @measurement_enabled
+          @logger = logger
+        end
+
+        private
+
+        attr_reader :measurement, :project, :namespace, :current_user, :file_path, :project_path, :logger
+
+        def measurement_enabled?
+          @measurement_enabled
+        end
+
+        def success(message)
+          logger.info(message)
+
+          true
+        end
+
+        def error(message)
+          logger.error(message)
+
+          false
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/export_task.rb b/lib/gitlab/import_export/project/export_task.rb
new file mode 100644
index 00000000000..ec287380c48
--- /dev/null
+++ b/lib/gitlab/import_export/project/export_task.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class ExportTask < BaseTask
+        def initialize(*)
+          super
+
+          @project = namespace.projects.find_by_path(@project_path)
+        end
+
+        def export
+          return error("Project with path: #{project_path} was not found. Please provide correct project path") unless project
+          return error("Invalid file path: #{file_path}. Please provide correct file path") unless file_path_exists?
+
+          with_export do
+            ::Projects::ImportExport::ExportService.new(project, current_user)
+              .execute(Gitlab::ImportExport::AfterExportStrategies::MoveFileStrategy.new(archive_path: file_path))
+          end
+
+          success('Done!')
+        end
+
+        private
+
+        def file_path_exists?
+          directory = File.dirname(file_path)
+
+          Dir.exist?(directory)
+        end
+
+        def with_export
+          with_request_store do
+            ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+              measurement_enabled? ? measurement.with_measuring { yield } : yield
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb
new file mode 100644
index 00000000000..ae654ddbeaf
--- /dev/null
+++ b/lib/gitlab/import_export/project/import_task.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    module Project
+      class ImportTask < BaseTask
+        def import
+          show_import_start_message
+
+          run_isolated_sidekiq_job
+
+          show_import_failures_count
+
+          return error(project.import_state.last_error) if project.import_state&.last_error
+          return error(project.errors.full_messages.to_sentence) if project.errors.any?
+
+          success('Done!')
+        end
+
+        private
+
+        # We want to ensure that all Sidekiq jobs are executed
+        # synchronously as part of that process.
+        # This ensures that all expensive operations do not escape
+        # to general Sidekiq clusters/nodes.
+        def with_isolated_sidekiq_job
+          Sidekiq::Testing.fake! do
+            with_request_store do
+              # If you are attempting to import a large project into a development environment,
+              # you may see Gitaly throw an error about too many calls or invocations.
+              # This is due to a n+1 calls limit being set for development setups (not enforced in production)
+              # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24475#note_283090635
+              # For development setups, this code-path will be excluded from n+1 detection.
+              ::Gitlab::GitalyClient.allow_n_plus_1_calls do
+                measurement_enabled? ? measurement.with_measuring { yield } : yield
+              end
+            end
+
+            true
+          end
+        end
+
+        def run_isolated_sidekiq_job
+          with_isolated_sidekiq_job do
+            @project = create_project
+
+            execute_sidekiq_job
+          end
+        end
+
+        def create_project
+          # We are disabling ObjectStorage for `import`
+          # as it is too slow to handle big archives:
+          # 1. DB transaction timeouts on upload
+          # 2. Download of archive before unpacking
+          disable_upload_object_storage do
+            service = Projects::GitlabProjectsImportService.new(
+              current_user,
+              {
+                namespace_id: namespace.id,
+                path:         project_path,
+                file:         File.open(file_path)
+              }
+            )
+
+            service.execute
+          end
+        end
+
+        def execute_sidekiq_job
+          Sidekiq::Worker.drain_all
+        end
+
+        def disable_upload_object_storage
+          overwrite_uploads_setting('background_upload', false) do
+            overwrite_uploads_setting('direct_upload', false) do
+              yield
+            end
+          end
+        end
+
+        def overwrite_uploads_setting(key, value)
+          old_value = Settings.uploads.object_store[key]
+          Settings.uploads.object_store[key] = value
+
+          yield
+
+        ensure
+          Settings.uploads.object_store[key] = old_value
+        end
+
+        def full_path
+          "#{namespace.full_path}/#{project_path}"
+        end
+
+        def show_import_start_message
+          logger.info "Importing GitLab export: #{file_path} into GitLab" \
+            " #{full_path}" \
+            " as #{current_user.name}"
+        end
+
+        def show_import_failures_count
+          return unless project.import_failures.exists?
+
+          logger.info "Total number of not imported relations: #{project.import_failures.count}"
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 8d81b2af065..09ed4eb568d 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -94,14 +94,6 @@ module Gitlab
         end
       end
 
-      def log_error(details)
-        @logger.error(log_base_data.merge(details))
-      end
-
-      def log_debug(details)
-        @logger.debug(log_base_data.merge(details))
-      end
-
       def log_base_data
         log = {
           importer:        'Import/Export',