Add latest changes from gitlab-org/gitlab@master

5 files changed, 297 insertions, 216 deletions

diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 2fd12e3aa78..9d04d55770d 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -5,6 +5,8 @@ module Gitlab
     class FileImporter
       include Gitlab::ImportExport::CommandLineUtil
 
+      ImporterError = Class.new(StandardError)
+
       MAX_RETRIES = 8
       IGNORED_FILENAMES = %w(. ..).freeze
 
@@ -12,8 +14,8 @@ module Gitlab
         new(*args).import
       end
 
-      def initialize(project:, archive_file:, shared:)
-        @project = project
+      def initialize(importable:, archive_file:, shared:)
+        @importable = importable
         @archive_file = archive_file
         @shared = shared
       end
@@ -52,7 +54,7 @@ module Gitlab
       def decompress_archive
         result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
 
-        raise Projects::ImportService::Error.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
+        raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
 
         result
       end
@@ -60,9 +62,9 @@ module Gitlab
       def copy_archive
         return if @archive_file
 
-        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @project))
+        @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable))
 
-        download_or_copy_upload(@project.import_export_upload.import_file, @archive_file)
+        download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file)
       end
 
       def remove_symlinks
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 62cf6c86906..a6463ed678c 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -39,7 +39,7 @@ module Gitlab
       end
 
       def import_file
-        Gitlab::ImportExport::FileImporter.import(project: project,
+        Gitlab::ImportExport::FileImporter.import(importable: project,
                                                   archive_file: archive_file,
                                                   shared: shared)
       end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 4e976cfca3a..d2e27388b51 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -3,10 +3,10 @@
 module Gitlab
   module ImportExport
     class MembersMapper
-      def initialize(exported_members:, user:, project:)
+      def initialize(exported_members:, user:, importable:)
         @exported_members = user.admin? ? exported_members : []
         @user = user
-        @project = project
+        @importable = importable
 
         # This needs to run first, as second call would be from #map
         # which means project members already exist.
@@ -19,7 +19,7 @@ module Gitlab
             @exported_members.inject(missing_keys_tracking_hash) do |hash, member|
               if member['user']
                 old_user_id = member['user']['id']
-                existing_user = User.where(find_project_user_query(member)).first
+                existing_user = User.where(find_user_query(member)).first
                 hash[old_user_id] = existing_user.id if existing_user && add_team_member(member, existing_user)
               else
                 add_team_member(member)
@@ -47,39 +47,48 @@ module Gitlab
       end
 
       def ensure_default_member!
-        @project.project_members.destroy_all # rubocop: disable DestroyAll
+        @importable.members.destroy_all # rubocop: disable DestroyAll
 
-        ProjectMember.create!(user: @user, access_level: ProjectMember::MAINTAINER, source_id: @project.id, importing: true)
+        relation_class.create!(user: @user, access_level: relation_class::MAINTAINER, source_id: @importable.id, importing: true)
       rescue => e
-        raise e, "Error adding importer user to project members. #{e.message}"
+        raise e, "Error adding importer user to #{@importable.class} members. #{e.message}"
       end
 
       def add_team_member(member, existing_user = nil)
         member['user'] = existing_user
 
-        ProjectMember.create(member_hash(member)).persisted?
+        relation_class.create(member_hash(member)).persisted?
       end
 
       def member_hash(member)
         parsed_hash(member).merge(
-          'source_id' => @project.id,
+          'source_id' => @importable.id,
           'importing' => true,
-          'access_level' => [member['access_level'], ProjectMember::MAINTAINER].min
+          'access_level' => [member['access_level'], relation_class::MAINTAINER].min
         ).except('user_id')
       end
 
       def parsed_hash(member)
-        Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys,
-                                                     relation_class: ProjectMember)
+        Gitlab::ImportExport::AttributeCleaner.clean(relation_hash:  member.deep_stringify_keys,
+                                                     relation_class: relation_class)
       end
 
-      def find_project_user_query(member)
+      def find_user_query(member)
         user_arel[:email].eq(member['user']['email']).or(user_arel[:username].eq(member['user']['username']))
       end
 
       def user_arel
         @user_arel ||= User.arel_table
       end
+
+      def relation_class
+        case @importable
+        when Project
+          ProjectMember
+        when Group
+          GroupMember
+        end
+      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 488e8e0fcea..e274b68a94f 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -3,9 +3,6 @@
 module Gitlab
   module ImportExport
     class ProjectTreeRestorer
-      # Relations which cannot be saved at project level (and have a group assigned)
-      GROUP_MODELS = [GroupLabel, Milestone].freeze
-
       attr_reader :user
       attr_reader :shared
       attr_reader :project
@@ -13,34 +10,23 @@ module Gitlab
       def initialize(user:, shared:, project:)
         @path = File.join(shared.export_path, 'project.json')
         @user = user
-        @shared = shared
+        @shared  = shared
         @project = project
       end
 
       def restore
-        begin
-          @tree_hash = read_tree_hash
-        rescue => e
-          Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
-          raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
-        end
-
+        @tree_hash = read_tree_hash
         @project_members = @tree_hash.delete('project_members')
 
         RelationRenameService.rename(@tree_hash)
 
-        ActiveRecord::Base.uncached do
-          ActiveRecord::Base.no_touching do
-            update_project_params!
-            create_project_relations!
-            post_import!
-          end
-        end
-
-        # ensure that we have latest version of the restore
-        @project.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+        if relation_tree_restorer.restore
+          @project.merge_requests.set_latest_merge_request_diff_ids!
 
-        true
+          true
+        else
+          false
+        end
       rescue => e
         @shared.error(e)
         false
@@ -51,195 +37,36 @@ module Gitlab
       def read_tree_hash
         json = IO.read(@path)
         ActiveSupport::JSON.decode(json)
-      end
-
-      def members_mapper
-        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
-                                                                    user: @user,
-                                                                    project: @project)
-      end
-
-      # A Hash of the imported merge request ID -> imported ID.
-      def merge_requests_mapping
-        @merge_requests_mapping ||= {}
-      end
-
-      # Loops through the tree of models defined in import_export.yml and
-      # finds them in the imported JSON so they can be instantiated and saved
-      # in the DB. The structure and relationships between models are guessed from
-      # the configuration yaml file too.
-      # Finally, it updates each attribute in the newly imported project.
-      def create_project_relations!
-        project_relations.each(&method(
-          :process_project_relation!))
-      end
-
-      def post_import!
-        @project.merge_requests.set_latest_merge_request_diff_ids!
-      end
-
-      def process_project_relation!(relation_key, relation_definition)
-        data_hashes = @tree_hash.delete(relation_key)
-        return unless data_hashes
-
-        # we do not care if we process array or hash
-        data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
-
-        relation_index = 0
-
-        # consume and remove objects from memory
-        while data_hash = data_hashes.shift
-          process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash)
-          relation_index += 1
-        end
-      end
-
-      def process_project_relation_item!(relation_key, relation_definition, relation_index, data_hash)
-        relation_object = build_relation(relation_key, relation_definition, data_hash)
-        return unless relation_object
-        return if group_model?(relation_object)
-
-        relation_object.project = @project
-        relation_object.save!
-
-        save_id_mapping(relation_key, data_hash, relation_object)
       rescue => e
-        # re-raise if not enabled
-        raise e unless Feature.enabled?(:import_graceful_failures, @project.group, default_enabled: true)
-
-        log_import_failure(relation_key, relation_index, e)
-      end
-
-      def log_import_failure(relation_key, relation_index, exception)
-        Gitlab::Sentry.track_acceptable_exception(exception,
-          extra: { project_id: @project.id, relation_key: relation_key, relation_index: relation_index })
-
-        ImportFailure.create(
-          project: @project,
-          relation_key: relation_key,
-          relation_index: relation_index,
-          exception_class: exception.class.to_s,
-          exception_message: exception.message.truncate(255),
-          correlation_id_value: Labkit::Correlation::CorrelationId.current_id
-        )
+        Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
+        raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
       end
 
-      # Older, serialized CI pipeline exports may only have a
-      # merge_request_id and not the full hash of the merge request. To
-      # import these pipelines, we need to preserve the mapping between
-      # the old and new the merge request ID.
-      def save_id_mapping(relation_key, data_hash, relation_object)
-        return unless relation_key == 'merge_requests'
-
-        merge_requests_mapping[data_hash['id']] = relation_object.id
-      end
-
-      def project_relations
-        @project_relations ||=
-          reader
-            .attributes_finder
-            .find_relations_tree(:project)
-            .deep_stringify_keys
-      end
-
-      def update_project_params!
-        project_params = @tree_hash.reject do |key, value|
-          project_relations.include?(key)
-        end
-
-        project_params = project_params.merge(
-          present_project_override_params)
-
-        # Cleaning all imported and overridden params
-        project_params = Gitlab::ImportExport::AttributeCleaner.clean(
-          relation_hash: project_params,
-          relation_class: Project,
-          excluded_keys: excluded_keys_for_relation(:project))
-
-        @project.assign_attributes(project_params)
-        @project.drop_visibility_level!
-
-        Gitlab::Timeless.timeless(@project) do
-          @project.save!
-        end
-      end
-
-      def present_project_override_params
-        # we filter out the empty strings from the overrides
-        # keeping the default values configured
-        project_override_params.transform_values do |value|
-          value.is_a?(String) ? value.presence : value
-        end.compact
-      end
-
-      def project_override_params
-        @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
-      end
-
-      def build_relations(relation_key, relation_definition, data_hashes)
-        data_hashes.map do |data_hash|
-          build_relation(relation_key, relation_definition, data_hash)
-        end.compact
-      end
-
-      def build_relation(relation_key, relation_definition, data_hash)
-        # TODO: This is hack to not create relation for the author
-        # Rather make `RelationFactory#set_note_author` to take care of that
-        return data_hash if relation_key == 'author'
-
-        # create relation objects recursively for all sub-objects
-        relation_definition.each do |sub_relation_key, sub_relation_definition|
-          transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
-        end
-
-        Gitlab::ImportExport::RelationFactory.create(
-          relation_sym: relation_key.to_sym,
-          relation_hash: data_hash,
-          members_mapper: members_mapper,
-          merge_requests_mapping: merge_requests_mapping,
+      def relation_tree_restorer
+        @relation_tree_restorer ||= RelationTreeRestorer.new(
           user: @user,
-          project: @project,
-          excluded_keys: excluded_keys_for_relation(relation_key))
+          shared: @shared,
+          importable: @project,
+          tree_hash: @tree_hash,
+          members_mapper: members_mapper,
+          relation_factory: relation_factory,
+          reader: reader
+        )
       end
 
-      def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
-        sub_data_hash = data_hash[sub_relation_key]
-        return unless sub_data_hash
-
-        # if object is a hash we can create simple object
-        # as it means that this is 1-to-1 vs 1-to-many
-        sub_data_hash =
-          if sub_data_hash.is_a?(Array)
-            build_relations(
-              sub_relation_key,
-              sub_relation_definition,
-              sub_data_hash).presence
-          else
-            build_relation(
-              sub_relation_key,
-              sub_relation_definition,
-              sub_data_hash)
-          end
-
-        # persist object(s) or delete from relation
-        if sub_data_hash
-          data_hash[sub_relation_key] = sub_data_hash
-        else
-          data_hash.delete(sub_relation_key)
-        end
+      def members_mapper
+        @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+                                                                    user: @user,
+                                                                    importable: @project)
       end
 
-      def group_model?(relation_object)
-        GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
+      def relation_factory
+        Gitlab::ImportExport::RelationFactory
       end
 
       def reader
         @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
       end
-
-      def excluded_keys_for_relation(relation)
-        reader.attributes_finder.find_excluded_keys(relation)
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
new file mode 100644
index 00000000000..15d1f8a8148
--- /dev/null
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -0,0 +1,243 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module ImportExport
+    class RelationTreeRestorer
+      # Relations which cannot be saved at project level (and have a group assigned)
+      GROUP_MODELS = [GroupLabel, Milestone].freeze
+
+      attr_reader :user
+      attr_reader :shared
+      attr_reader :importable
+      attr_reader :tree_hash
+
+      def initialize(user:, shared:, importable:, tree_hash:, members_mapper:, relation_factory:, reader:)
+        @user = user
+        @shared = shared
+        @importable = importable
+        @tree_hash = tree_hash
+        @members_mapper = members_mapper
+        @relation_factory = relation_factory
+        @reader = reader
+      end
+
+      def restore
+        ActiveRecord::Base.uncached do
+          ActiveRecord::Base.no_touching do
+            update_params!
+            create_relations!
+          end
+        end
+
+        # ensure that we have latest version of the restore
+        @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      # Loops through the tree of models defined in import_export.yml and
+      # finds them in the imported JSON so they can be instantiated and saved
+      # in the DB. The structure and relationships between models are guessed from
+      # the configuration yaml file too.
+      # Finally, it updates each attribute in the newly imported project/group.
+      def create_relations!
+        relations.each(&method(:process_relation!))
+      end
+
+      def process_relation!(relation_key, relation_definition)
+        data_hashes = @tree_hash.delete(relation_key)
+        return unless data_hashes
+
+        # we do not care if we process array or hash
+        data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
+
+        relation_index = 0
+
+        # consume and remove objects from memory
+        while data_hash = data_hashes.shift
+          process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
+          relation_index += 1
+        end
+      end
+
+      def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
+        relation_object = build_relation(relation_key, relation_definition, data_hash)
+        return unless relation_object
+        return if importable_class == Project && group_model?(relation_object)
+
+        relation_object.assign_attributes(importable_class_sym => @importable)
+        relation_object.save!
+
+        save_id_mapping(relation_key, data_hash, relation_object)
+      rescue => e
+        # re-raise if not enabled
+        raise e unless Feature.enabled?(:import_graceful_failures, @importable.group, default_enabled: true)
+
+        log_import_failure(relation_key, relation_index, e)
+      end
+
+      def log_import_failure(relation_key, relation_index, exception)
+        Gitlab::Sentry.track_acceptable_exception(
+          exception,
+          extra: { project_id: @importable.id,
+                   relation_key: relation_key,
+                   relation_index: relation_index })
+
+        ImportFailure.create(
+          project: @importable,
+          relation_key: relation_key,
+          relation_index: relation_index,
+          exception_class: exception.class.to_s,
+          exception_message: exception.message.truncate(255),
+          correlation_id_value: Labkit::Correlation::CorrelationId.current_id
+        )
+      end
+
+      # Older, serialized CI pipeline exports may only have a
+      # merge_request_id and not the full hash of the merge request. To
+      # import these pipelines, we need to preserve the mapping between
+      # the old and new the merge request ID.
+      def save_id_mapping(relation_key, data_hash, relation_object)
+        return unless importable_class == Project
+        return unless relation_key == 'merge_requests'
+
+        merge_requests_mapping[data_hash['id']] = relation_object.id
+      end
+
+      def relations
+        @relations ||=
+          @reader
+            .attributes_finder
+            .find_relations_tree(importable_class_sym)
+            .deep_stringify_keys
+      end
+
+      def update_params!
+        params = @tree_hash.reject do |key, _|
+          relations.include?(key)
+        end
+
+        params = params.merge(present_override_params)
+
+        # Cleaning all imported and overridden params
+        params = Gitlab::ImportExport::AttributeCleaner.clean(
+          relation_hash:  params,
+          relation_class: importable_class,
+          excluded_keys:  excluded_keys_for_relation(importable_class_sym))
+
+        @importable.assign_attributes(params)
+        @importable.drop_visibility_level! if importable_class == Project
+
+        Gitlab::Timeless.timeless(@importable) do
+          @importable.save!
+        end
+      end
+
+      def present_override_params
+        # we filter out the empty strings from the overrides
+        # keeping the default values configured
+        override_params&.transform_values do |value|
+          value.is_a?(String) ? value.presence : value
+        end&.compact
+      end
+
+      def override_params
+        @importable_override_params ||= importable_override_params
+      end
+
+      def importable_override_params
+        if @importable.respond_to?(:import_data)
+          @importable.import_data&.data&.fetch('override_params', nil) || {}
+        else
+          {}
+        end
+      end
+
+      def build_relations(relation_key, relation_definition, data_hashes)
+        data_hashes.map do |data_hash|
+          build_relation(relation_key, relation_definition, data_hash)
+        end.compact
+      end
+
+      def build_relation(relation_key, relation_definition, data_hash)
+        # TODO: This is hack to not create relation for the author
+        # Rather make `RelationFactory#set_note_author` to take care of that
+        return data_hash if relation_key == 'author'
+
+        # create relation objects recursively for all sub-objects
+        relation_definition.each do |sub_relation_key, sub_relation_definition|
+          transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+        end
+
+        @relation_factory.create(relation_factory_params(relation_key, data_hash))
+      end
+
+      def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+        sub_data_hash = data_hash[sub_relation_key]
+        return unless sub_data_hash
+
+        # if object is a hash we can create simple object
+        # as it means that this is 1-to-1 vs 1-to-many
+        sub_data_hash =
+          if sub_data_hash.is_a?(Array)
+            build_relations(
+              sub_relation_key,
+              sub_relation_definition,
+              sub_data_hash).presence
+          else
+            build_relation(
+              sub_relation_key,
+              sub_relation_definition,
+              sub_data_hash)
+          end
+
+        # persist object(s) or delete from relation
+        if sub_data_hash
+          data_hash[sub_relation_key] = sub_data_hash
+        else
+          data_hash.delete(sub_relation_key)
+        end
+      end
+
+      def group_model?(relation_object)
+        GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
+      end
+
+      def excluded_keys_for_relation(relation)
+        @reader.attributes_finder.find_excluded_keys(relation)
+      end
+
+      def importable_class
+        @importable.class
+      end
+
+      def importable_class_sym
+        importable_class.to_s.downcase.to_sym
+      end
+
+      # A Hash of the imported merge request ID -> imported ID.
+      def merge_requests_mapping
+        @merge_requests_mapping ||= {}
+      end
+
+      def relation_factory_params(relation_key, data_hash)
+        base_params = {
+          relation_sym:   relation_key.to_sym,
+          relation_hash:  data_hash,
+          members_mapper: @members_mapper,
+          user:           @user,
+          excluded_keys:  excluded_keys_for_relation(relation_key)
+        }
+
+        base_params[:merge_requests_mapping] = merge_requests_mapping if importable_class == Project
+        base_params[importable_class_sym] = @importable
+        base_params
+      end
+    end
+  end
+end