Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2020-02-26 21:09:24 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2020-02-26 21:09:24 +0300
commit619d0b6922a6cf95d291fbbf5fa3d09e772a1ea8 (patch)
treefb8f8e036cec1b32166206bb5102af6c5dca8cfe /lib/gitlab
parent17ab40ca089e1aef61a83f77ab6df62a72f6ce06 (diff)
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib/gitlab')
-rw-r--r--lib/gitlab/checks/branch_check.rb18
-rw-r--r--lib/gitlab/checks/diff_check.rb4
-rw-r--r--lib/gitlab/checks/lfs_check.rb2
-rw-r--r--lib/gitlab/checks/push_check.rb2
-rw-r--r--lib/gitlab/checks/tag_check.rb8
-rw-r--r--lib/gitlab/git/repository.rb1
-rw-r--r--lib/gitlab/git_access.rb30
-rw-r--r--lib/gitlab/git_access_wiki.rb4
-rw-r--r--lib/gitlab/gitaly_client/commit_service.rb3
-rw-r--r--lib/gitlab/import_export.rb4
-rw-r--r--lib/gitlab/import_export/attribute_cleaner.rb4
-rw-r--r--lib/gitlab/import_export/base/object_builder.rb105
-rw-r--r--lib/gitlab/import_export/base/relation_factory.rb308
-rw-r--r--lib/gitlab/import_export/base_object_builder.rb103
-rw-r--r--lib/gitlab/import_export/base_relation_factory.rb306
-rw-r--r--lib/gitlab/import_export/group/import_export.yml (renamed from lib/gitlab/import_export/group_import_export.yml)0
-rw-r--r--lib/gitlab/import_export/group/object_builder.rb57
-rw-r--r--lib/gitlab/import_export/group/relation_factory.rb42
-rw-r--r--lib/gitlab/import_export/group/tree_restorer.rb118
-rw-r--r--lib/gitlab/import_export/group/tree_saver.rb57
-rw-r--r--lib/gitlab/import_export/group_object_builder.rb55
-rw-r--r--lib/gitlab/import_export/group_project_object_builder.rb117
-rw-r--r--lib/gitlab/import_export/group_relation_factory.rb40
-rw-r--r--lib/gitlab/import_export/group_tree_restorer.rb116
-rw-r--r--lib/gitlab/import_export/group_tree_saver.rb55
-rw-r--r--lib/gitlab/import_export/importer.rb4
-rw-r--r--lib/gitlab/import_export/members_mapper.rb4
-rw-r--r--lib/gitlab/import_export/project/import_export.yml (renamed from lib/gitlab/import_export/import_export.yml)0
-rw-r--r--lib/gitlab/import_export/project/object_builder.rb119
-rw-r--r--lib/gitlab/import_export/project/relation_factory.rb162
-rw-r--r--lib/gitlab/import_export/project/tree_loader.rb74
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb94
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb70
-rw-r--r--lib/gitlab/import_export/project_relation_factory.rb160
-rw-r--r--lib/gitlab/import_export/project_tree_loader.rb72
-rw-r--r--lib/gitlab/import_export/project_tree_restorer.rb92
-rw-r--r--lib/gitlab/import_export/project_tree_saver.rb68
-rw-r--r--lib/gitlab/import_export/relation_tree_restorer.rb4
38 files changed, 1253 insertions, 1229 deletions
diff --git a/lib/gitlab/checks/branch_check.rb b/lib/gitlab/checks/branch_check.rb
index 4ddc1c718c7..7be0ef05a49 100644
--- a/lib/gitlab/checks/branch_check.rb
+++ b/lib/gitlab/checks/branch_check.rb
@@ -28,7 +28,7 @@ module Gitlab
logger.log_timed(LOG_MESSAGES[:delete_default_branch_check]) do
if deletion? && branch_name == project.default_branch
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_default_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_default_branch]
end
end
@@ -42,7 +42,7 @@ module Gitlab
return unless ProtectedBranch.protected?(project, branch_name) # rubocop:disable Cop/AvoidReturnFromBlocks
if forced_push?
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:force_push_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:force_push_protected_branch]
end
end
@@ -62,15 +62,15 @@ module Gitlab
break if user_access.can_push_to_branch?(branch_name)
unless user_access.can_merge_to_branch?(branch_name)
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_branch]
end
unless safe_commit_for_new_protected_branch?
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:invalid_commit_create_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:invalid_commit_create_protected_branch]
end
unless updated_from_web?
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_create_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_create_protected_branch]
end
end
end
@@ -78,11 +78,11 @@ module Gitlab
def protected_branch_deletion_checks
logger.log_timed(LOG_MESSAGES[:protected_branch_deletion_checks]) do
unless user_access.can_delete_branch?(branch_name)
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_master_delete_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_master_delete_protected_branch]
end
unless updated_from_web?
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:non_web_delete_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:non_web_delete_protected_branch]
end
end
end
@@ -91,11 +91,11 @@ module Gitlab
logger.log_timed(LOG_MESSAGES[:protected_branch_push_checks]) do
if matching_merge_request?
unless user_access.can_merge_to_branch?(branch_name) || user_access.can_push_to_branch?(branch_name)
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:merge_protected_branch]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:merge_protected_branch]
end
else
unless user_access.can_push_to_branch?(branch_name)
- raise GitAccess::UnauthorizedError, push_to_protected_branch_rejected_message
+ raise GitAccess::ForbiddenError, push_to_protected_branch_rejected_message
end
end
end
diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb
index 5de71addd5f..0eb2b4c79ef 100644
--- a/lib/gitlab/checks/diff_check.rb
+++ b/lib/gitlab/checks/diff_check.rb
@@ -46,7 +46,7 @@ module Gitlab
def validate_diff(diff)
validations_for_diff.each do |validation|
if error = validation.call(diff)
- raise ::Gitlab::GitAccess::UnauthorizedError, error
+ raise ::Gitlab::GitAccess::ForbiddenError, error
end
end
end
@@ -77,7 +77,7 @@ module Gitlab
logger.log_timed(LOG_MESSAGES[__method__]) do
path_validations.each do |validation|
if error = validation.call(file_paths)
- raise ::Gitlab::GitAccess::UnauthorizedError, error
+ raise ::Gitlab::GitAccess::ForbiddenError, error
end
end
end
diff --git a/lib/gitlab/checks/lfs_check.rb b/lib/gitlab/checks/lfs_check.rb
index 7b013567a03..f81c215d847 100644
--- a/lib/gitlab/checks/lfs_check.rb
+++ b/lib/gitlab/checks/lfs_check.rb
@@ -15,7 +15,7 @@ module Gitlab
lfs_check = Checks::LfsIntegrity.new(project, newrev, logger.time_left)
if lfs_check.objects_missing?
- raise GitAccess::UnauthorizedError, ERROR_MESSAGE
+ raise GitAccess::ForbiddenError, ERROR_MESSAGE
end
end
end
diff --git a/lib/gitlab/checks/push_check.rb b/lib/gitlab/checks/push_check.rb
index 91f8d0bdbc8..7cc5bc56cbb 100644
--- a/lib/gitlab/checks/push_check.rb
+++ b/lib/gitlab/checks/push_check.rb
@@ -6,7 +6,7 @@ module Gitlab
def validate!
logger.log_timed("Checking if you are allowed to push...") do
unless can_push?
- raise GitAccess::UnauthorizedError, GitAccess::ERROR_MESSAGES[:push_code]
+ raise GitAccess::ForbiddenError, GitAccess::ERROR_MESSAGES[:push_code]
end
end
end
diff --git a/lib/gitlab/checks/tag_check.rb b/lib/gitlab/checks/tag_check.rb
index ced0612a7a3..a47e55cb160 100644
--- a/lib/gitlab/checks/tag_check.rb
+++ b/lib/gitlab/checks/tag_check.rb
@@ -20,7 +20,7 @@ module Gitlab
logger.log_timed(LOG_MESSAGES[:tag_checks]) do
if tag_exists? && user_access.cannot_do_action?(:admin_tag)
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:change_existing_tags]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:change_existing_tags]
end
end
@@ -33,11 +33,11 @@ module Gitlab
logger.log_timed(LOG_MESSAGES[__method__]) do
return unless ProtectedTag.protected?(project, tag_name) # rubocop:disable Cop/AvoidReturnFromBlocks
- raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:update_protected_tag]) if update?
- raise(GitAccess::UnauthorizedError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
+ raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:update_protected_tag]) if update?
+ raise(GitAccess::ForbiddenError, ERROR_MESSAGES[:delete_protected_tag]) if deletion?
unless user_access.can_create_tag?(tag_name)
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:create_protected_tag]
+ raise GitAccess::ForbiddenError, ERROR_MESSAGES[:create_protected_tag]
end
end
end
diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb
index 6bfe744a5cd..29324381cb5 100644
--- a/lib/gitlab/git/repository.rb
+++ b/lib/gitlab/git/repository.rb
@@ -322,6 +322,7 @@ module Gitlab
limit: 10,
offset: 0,
path: nil,
+ author: nil,
follow: false,
skip_merges: false,
after: nil,
diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb
index 906350e57c5..d6c87b858a8 100644
--- a/lib/gitlab/git_access.rb
+++ b/lib/gitlab/git_access.rb
@@ -6,7 +6,7 @@ module Gitlab
class GitAccess
include Gitlab::Utils::StrongMemoize
- UnauthorizedError = Class.new(StandardError)
+ ForbiddenError = Class.new(StandardError)
NotFoundError = Class.new(StandardError)
ProjectCreationError = Class.new(StandardError)
TimeoutError = Class.new(StandardError)
@@ -125,7 +125,7 @@ module Gitlab
return unless actor.is_a?(Key)
unless actor.valid?
- raise UnauthorizedError, "Your SSH key #{actor.errors[:key].first}."
+ raise ForbiddenError, "Your SSH key #{actor.errors[:key].first}."
end
end
@@ -133,7 +133,7 @@ module Gitlab
return if request_from_ci_build?
unless protocol_allowed?
- raise UnauthorizedError, "Git access over #{protocol.upcase} is not allowed"
+ raise ForbiddenError, "Git access over #{protocol.upcase} is not allowed"
end
end
@@ -148,7 +148,7 @@ module Gitlab
unless user_access.allowed?
message = Gitlab::Auth::UserAccessDeniedReason.new(user).rejection_message
- raise UnauthorizedError, message
+ raise ForbiddenError, message
end
end
@@ -156,11 +156,11 @@ module Gitlab
case cmd
when *DOWNLOAD_COMMANDS
unless authentication_abilities.include?(:download_code) || authentication_abilities.include?(:build_download_code)
- raise UnauthorizedError, ERROR_MESSAGES[:auth_download]
+ raise ForbiddenError, ERROR_MESSAGES[:auth_download]
end
when *PUSH_COMMANDS
unless authentication_abilities.include?(:push_code)
- raise UnauthorizedError, ERROR_MESSAGES[:auth_upload]
+ raise ForbiddenError, ERROR_MESSAGES[:auth_upload]
end
end
end
@@ -189,19 +189,19 @@ module Gitlab
def check_upload_pack_disabled!
if http? && upload_pack_disabled_over_http?
- raise UnauthorizedError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
+ raise ForbiddenError, ERROR_MESSAGES[:upload_pack_disabled_over_http]
end
end
def check_receive_pack_disabled!
if http? && receive_pack_disabled_over_http?
- raise UnauthorizedError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
+ raise ForbiddenError, ERROR_MESSAGES[:receive_pack_disabled_over_http]
end
end
def check_command_existence!(cmd)
unless ALL_COMMANDS.include?(cmd)
- raise UnauthorizedError, ERROR_MESSAGES[:command_not_allowed]
+ raise ForbiddenError, ERROR_MESSAGES[:command_not_allowed]
end
end
@@ -209,7 +209,7 @@ module Gitlab
return unless receive_pack?(cmd)
if Gitlab::Database.read_only?
- raise UnauthorizedError, push_to_read_only_message
+ raise ForbiddenError, push_to_read_only_message
end
end
@@ -253,23 +253,23 @@ module Gitlab
guest_can_download_code?
unless passed
- raise UnauthorizedError, ERROR_MESSAGES[:download]
+ raise ForbiddenError, ERROR_MESSAGES[:download]
end
end
def check_push_access!
if project.repository_read_only?
- raise UnauthorizedError, ERROR_MESSAGES[:read_only]
+ raise ForbiddenError, ERROR_MESSAGES[:read_only]
end
if deploy_key?
unless deploy_key.can_push_to?(project)
- raise UnauthorizedError, ERROR_MESSAGES[:deploy_key_upload]
+ raise ForbiddenError, ERROR_MESSAGES[:deploy_key_upload]
end
elsif user
# User access is verified in check_change_access!
else
- raise UnauthorizedError, ERROR_MESSAGES[:upload]
+ raise ForbiddenError, ERROR_MESSAGES[:upload]
end
check_change_access!
@@ -284,7 +284,7 @@ module Gitlab
project.any_branch_allows_collaboration?(user_access.user)
unless can_push
- raise GitAccess::UnauthorizedError, ERROR_MESSAGES[:push_code]
+ raise ForbiddenError, ERROR_MESSAGES[:push_code]
end
else
# If there are worktrees with a HEAD pointing to a non-existent object,
diff --git a/lib/gitlab/git_access_wiki.rb b/lib/gitlab/git_access_wiki.rb
index 3d0db753f6e..aad46937c32 100644
--- a/lib/gitlab/git_access_wiki.rb
+++ b/lib/gitlab/git_access_wiki.rb
@@ -19,11 +19,11 @@ module Gitlab
def check_change_access!
unless user_access.can_do_action?(:create_wiki)
- raise UnauthorizedError, ERROR_MESSAGES[:write_to_wiki]
+ raise ForbiddenError, ERROR_MESSAGES[:write_to_wiki]
end
if Gitlab::Database.read_only?
- raise UnauthorizedError, push_to_read_only_message
+ raise ForbiddenError, push_to_read_only_message
end
true
diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb
index ac22f5bf419..1f914dc95d1 100644
--- a/lib/gitlab/gitaly_client/commit_service.rb
+++ b/lib/gitlab/gitaly_client/commit_service.rb
@@ -324,7 +324,8 @@ module Gitlab
request.after = GitalyClient.timestamp(options[:after]) if options[:after]
request.before = GitalyClient.timestamp(options[:before]) if options[:before]
request.revision = encode_binary(options[:ref]) if options[:ref]
- request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present?
+ request.author = encode_binary(options[:author]) if options[:author]
+ request.order = options[:order].upcase.sub('DEFAULT', 'NONE') if options[:order].present?
request.paths = encode_repeated(Array(options[:path])) if options[:path].present?
diff --git a/lib/gitlab/import_export.rb b/lib/gitlab/import_export.rb
index 8ce6549c0c7..1033e6c4e05 100644
--- a/lib/gitlab/import_export.rb
+++ b/lib/gitlab/import_export.rb
@@ -43,7 +43,7 @@ module Gitlab
end
def config_file
- Rails.root.join('lib/gitlab/import_export/import_export.yml')
+ Rails.root.join('lib/gitlab/import_export/project/import_export.yml')
end
def version_filename
@@ -77,7 +77,7 @@ module Gitlab
end
def group_config_file
- Rails.root.join('lib/gitlab/import_export/group_import_export.yml')
+ Rails.root.join('lib/gitlab/import_export/group/import_export.yml')
end
end
end
diff --git a/lib/gitlab/import_export/attribute_cleaner.rb b/lib/gitlab/import_export/attribute_cleaner.rb
index d1c20dff799..3bfc059dcd3 100644
--- a/lib/gitlab/import_export/attribute_cleaner.rb
+++ b/lib/gitlab/import_export/attribute_cleaner.rb
@@ -4,8 +4,8 @@ module Gitlab
module ImportExport
class AttributeCleaner
ALLOWED_REFERENCES = [
- *ProjectRelationFactory::PROJECT_REFERENCES,
- *ProjectRelationFactory::USER_REFERENCES,
+ *Gitlab::ImportExport::Project::RelationFactory::PROJECT_REFERENCES,
+ *Gitlab::ImportExport::Project::RelationFactory::USER_REFERENCES,
'group_id',
'commit_id',
'discussion_id',
diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb
new file mode 100644
index 00000000000..109d2e233a5
--- /dev/null
+++ b/lib/gitlab/import_export/base/object_builder.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Base
+ # Base class for Group & Project Object Builders.
+ # This class is not intended to be used on its own but
+ # rather inherited from.
+ #
+ # Cache keeps 1000 entries at most, 1000 is chosen based on:
+ # - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
+ # (leave some buffer it should be less than 1M). It is afforable cost for project import.
+ # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
+ # For example, gitlab has ~970 labels and 26 milestones.
+ LRU_CACHE_SIZE = 1000
+
+ class ObjectBuilder
+ def self.build(*args)
+ new(*args).find
+ end
+
+ def initialize(klass, attributes)
+ @klass = klass.ancestors.include?(Label) ? Label : klass
+ @attributes = attributes
+
+ if Gitlab::SafeRequestStore.active?
+ @lru_cache = cache_from_request_store
+ @cache_key = [klass, attributes]
+ end
+ end
+
+ def find
+ find_with_cache do
+ find_object || klass.create(prepare_attributes)
+ end
+ end
+
+ protected
+
+ def where_clauses
+ raise NotImplementedError
+ end
+
+ # attributes wrapped in a method to be
+ # adjusted in sub-class if needed
+ def prepare_attributes
+ attributes
+ end
+
+ private
+
+ attr_reader :klass, :attributes, :lru_cache, :cache_key
+
+ def find_with_cache
+ return yield unless lru_cache && cache_key
+
+ lru_cache[cache_key] ||= yield
+ end
+
+ def cache_from_request_store
+ Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
+ end
+
+ def find_object
+ klass.where(where_clause).first
+ end
+
+ def where_clause
+ where_clauses.reduce(:and)
+ end
+
+ def table
+ @table ||= klass.arel_table
+ end
+
+ # Returns Arel clause:
+ # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
+ # from the given Hash of attributes.
+ def attrs_to_arel(attrs)
+ attrs.map do |key, value|
+ table[key].eq(value)
+ end.reduce(:and)
+ end
+
+ # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
+ # if attributes has 'title key, otherwise `nil`.
+ def where_clause_for_title
+ attrs_to_arel(attributes.slice('title'))
+ end
+
+ # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
+ # if attributes has 'description key, otherwise `nil`.
+ def where_clause_for_description
+ attrs_to_arel(attributes.slice('description'))
+ end
+
+ # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
+ # if attributes has 'created_at key, otherwise `nil`.
+ def where_clause_for_created_at
+ attrs_to_arel(attributes.slice('created_at'))
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
new file mode 100644
index 00000000000..688627d1f2f
--- /dev/null
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Base
+ class RelationFactory
+ include Gitlab::Utils::StrongMemoize
+
+ IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
+
+ OVERRIDES = {}.freeze
+ EXISTING_OBJECT_RELATIONS = %i[].freeze
+
+ # This represents all relations that have unique key on `project_id` or `group_id`
+ UNIQUE_RELATIONS = %i[].freeze
+
+ USER_REFERENCES = %w[
+ author_id
+ assignee_id
+ updated_by_id
+ merged_by_id
+ latest_closed_by_id
+ user_id
+ created_by_id
+ last_edited_by_id
+ merge_user_id
+ resolved_by_id
+ closed_by_id
+ owner_id
+ ].freeze
+
+ TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
+
+ def self.create(*args)
+ new(*args).create
+ end
+
+ def self.relation_class(relation_name)
+ # There are scenarios where the model is pluralized (e.g.
+ # MergeRequest::Metrics), and we don't want to force it to singular
+ # with #classify.
+ relation_name.to_s.classify.constantize
+ rescue NameError
+ relation_name.to_s.constantize
+ end
+
+ def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: [])
+ @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
+ @relation_hash = relation_hash.except('noteable_id')
+ @members_mapper = members_mapper
+ @object_builder = object_builder
+ @user = user
+ @importable = importable
+ @imported_object_retries = 0
+ @relation_hash[importable_column_name] = @importable.id
+
+ # Remove excluded keys from relation_hash
+ # We don't do this in the parsed_relation_hash because of the 'transformed attributes'
+ # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then,
+ # in the create method that attribute is renamed to diff. And because diff is an excluded key,
+ # if we clean the excluded keys in the parsed_relation_hash, it will be removed
+ # from the object attributes and the export will fail.
+ @relation_hash.except!(*excluded_keys)
+ end
+
+ # Creates an object from an actual model with name "relation_sym" with params from
+ # the relation_hash, updating references with new object IDs, mapping users using
+ # the "members_mapper" object, also updating notes if required.
+ def create
+ return if invalid_relation?
+
+ setup_base_models
+ setup_models
+
+ generate_imported_object
+ end
+
+ def self.overrides
+ self::OVERRIDES
+ end
+
+ def self.existing_object_relations
+ self::EXISTING_OBJECT_RELATIONS
+ end
+
+ private
+
+ def invalid_relation?
+ false
+ end
+
+ def setup_models
+ raise NotImplementedError
+ end
+
+ def unique_relations
+ # define in sub-class if any
+ self.class::UNIQUE_RELATIONS
+ end
+
+ def setup_base_models
+ update_user_references
+ remove_duplicate_assignees
+ reset_tokens!
+ remove_encrypted_attributes!
+ end
+
+ def update_user_references
+ self.class::USER_REFERENCES.each do |reference|
+ if @relation_hash[reference]
+ @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
+ end
+ end
+ end
+
+ def remove_duplicate_assignees
+ return unless @relation_hash['issue_assignees']
+
+ # When an assignee did not exist in the members mapper, the importer is
+ # assigned. We only need to assign each user once.
+ @relation_hash['issue_assignees'].uniq!(&:user_id)
+ end
+
+ def generate_imported_object
+ imported_object
+ end
+
+ def reset_tokens!
+ return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name)
+
+ # If we import/export to the same instance, tokens will have to be reset.
+ # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
+ relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
+ @relation_hash[token] = nil
+ end
+ end
+
+ def remove_encrypted_attributes!
+ return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
+
+ relation_class.encrypted_attributes.each_key do |key|
+ @relation_hash[key.to_s] = nil
+ end
+ end
+
+ def relation_class
+ @relation_class ||= self.class.relation_class(@relation_name)
+ end
+
+ def importable_column_name
+ importable_class_name.concat('_id')
+ end
+
+ def importable_class_name
+ @importable.class.to_s.downcase
+ end
+
+ def imported_object
+ if existing_or_new_object.respond_to?(:importing)
+ existing_or_new_object.importing = true
+ end
+
+ existing_or_new_object
+ rescue ActiveRecord::RecordNotUnique
+ # as the operation is not atomic, retry in the unlikely scenario an INSERT is
+ # performed on the same object between the SELECT and the INSERT
+ @imported_object_retries += 1
+ retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
+ end
+
+ def parsed_relation_hash
+ @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
+ relation_class: relation_class)
+ end
+
+ def existing_or_new_object
+ # Only find existing records to avoid mapping tables such as milestones
+ # Otherwise always create the record, skipping the extra SELECT clause.
+ @existing_or_new_object ||= begin
+ if existing_object?
+ attribute_hash = attribute_hash_for(['events'])
+
+ existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
+
+ existing_object
+ else
+ # Because of single-type inheritance, we need to be careful to use the `type` field
+ # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497
+ inheritance_column = relation_class.try(:inheritance_column)
+ inheritance_attributes = parsed_relation_hash.slice(inheritance_column)
+ object = relation_class.new(inheritance_attributes)
+ object.assign_attributes(parsed_relation_hash)
+ object
+ end
+ end
+ end
+
+ def attribute_hash_for(attributes)
+ attributes.each_with_object({}) do |hash, value|
+ hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
+ hash
+ end
+ end
+
+ def existing_object
+ @existing_object ||= find_or_create_object!
+ end
+
+ def unique_relation_object
+ unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id)
+ unique_relation_object.assign_attributes(parsed_relation_hash)
+ unique_relation_object
+ end
+
+ def find_or_create_object!
+ return unique_relation_object if unique_relation?
+
+ # Can't use IDs as validation exists calling `group` or `project` attributes
+ finder_hash = parsed_relation_hash.tap do |hash|
+ if relation_class.attribute_method?('group_id') && @importable.is_a?(::Project)
+ hash['group'] = @importable.group
+ end
+
+ hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym)
+ hash.delete(importable_column_name)
+ end
+
+ @object_builder.build(relation_class, finder_hash)
+ end
+
+ def setup_note
+ set_note_author
+ # attachment is deprecated and note uploads are handled by Markdown uploader
+ @relation_hash['attachment'] = nil
+ end
+
+ # Sets the author for a note. If the user importing the project
+ # has admin access, an actual mapping with new project members
+ # will be used. Otherwise, a note stating the original author name
+ # is left.
+ def set_note_author
+ old_author_id = @relation_hash['author_id']
+ author = @relation_hash.delete('author')
+
+ update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
+ end
+
+ def has_author?(old_author_id)
+ admin_user? && @members_mapper.include?(old_author_id)
+ end
+
+ def missing_author_note(updated_at, author_name)
+ timestamp = updated_at.split('.').first
+ "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
+ end
+
+ def update_note_for_missing_author(author_name)
+ @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
+ @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}"
+ end
+
+ def admin_user?
+ @user.admin?
+ end
+
+ def existing_object?
+ strong_memoize(:_existing_object) do
+ self.class.existing_object_relations.include?(@relation_name) || unique_relation?
+ end
+ end
+
+ def unique_relation?
+ strong_memoize(:unique_relation) do
+ importable_foreign_key.present? &&
+ (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?)
+ end
+ end
+
+ def has_unique_index_on_importable_fk?
+ cache = cached_has_unique_index_on_importable_fk
+ table_name = relation_class.table_name
+ return cache[table_name] if cache.has_key?(table_name)
+
+ index_exists =
+ ActiveRecord::Base.connection.index_exists?(
+ relation_class.table_name,
+ importable_foreign_key,
+ unique: true)
+
+ cache[table_name] = index_exists
+ end
+
+ # Avoid unnecessary DB requests
+ def cached_has_unique_index_on_importable_fk
+ Thread.current[:cached_has_unique_index_on_importable_fk] ||= {}
+ end
+
+ def uses_importable_fk_as_primary_key?
+ relation_class.primary_key == importable_foreign_key
+ end
+
+ def importable_foreign_key
+ relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/base_object_builder.rb b/lib/gitlab/import_export/base_object_builder.rb
deleted file mode 100644
index ec66b7a7a4f..00000000000
--- a/lib/gitlab/import_export/base_object_builder.rb
+++ /dev/null
@@ -1,103 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- # Base class for Group & Project Object Builders.
- # This class is not intended to be used on its own but
- # rather inherited from.
- #
- # Cache keeps 1000 entries at most, 1000 is chosen based on:
- # - one cache entry uses around 0.5K memory, 1000 items uses around 500K.
- # (leave some buffer it should be less than 1M). It is afforable cost for project import.
- # - for projects in Gitlab.com, it seems 1000 entries for labels/milestones is enough.
- # For example, gitlab has ~970 labels and 26 milestones.
- LRU_CACHE_SIZE = 1000
-
- class BaseObjectBuilder
- def self.build(*args)
- new(*args).find
- end
-
- def initialize(klass, attributes)
- @klass = klass.ancestors.include?(Label) ? Label : klass
- @attributes = attributes
-
- if Gitlab::SafeRequestStore.active?
- @lru_cache = cache_from_request_store
- @cache_key = [klass, attributes]
- end
- end
-
- def find
- find_with_cache do
- find_object || klass.create(prepare_attributes)
- end
- end
-
- protected
-
- def where_clauses
- raise NotImplementedError
- end
-
- # attributes wrapped in a method to be
- # adjusted in sub-class if needed
- def prepare_attributes
- attributes
- end
-
- private
-
- attr_reader :klass, :attributes, :lru_cache, :cache_key
-
- def find_with_cache
- return yield unless lru_cache && cache_key
-
- lru_cache[cache_key] ||= yield
- end
-
- def cache_from_request_store
- Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE)
- end
-
- def find_object
- klass.where(where_clause).first
- end
-
- def where_clause
- where_clauses.reduce(:and)
- end
-
- def table
- @table ||= klass.arel_table
- end
-
- # Returns Arel clause:
- # `"{table_name}"."{attrs.keys[0]}" = '{attrs.values[0]} AND {table_name}"."{attrs.keys[1]}" = '{attrs.values[1]}"`
- # from the given Hash of attributes.
- def attrs_to_arel(attrs)
- attrs.map do |key, value|
- table[key].eq(value)
- end.reduce(:and)
- end
-
- # Returns Arel clause `"{table_name}"."title" = '{attributes['title']}'`
- # if attributes has 'title key, otherwise `nil`.
- def where_clause_for_title
- attrs_to_arel(attributes.slice('title'))
- end
-
- # Returns Arel clause `"{table_name}"."description" = '{attributes['description']}'`
- # if attributes has 'description key, otherwise `nil`.
- def where_clause_for_description
- attrs_to_arel(attributes.slice('description'))
- end
-
- # Returns Arel clause `"{table_name}"."created_at" = '{attributes['created_at']}'`
- # if attributes has 'created_at key, otherwise `nil`.
- def where_clause_for_created_at
- attrs_to_arel(attributes.slice('created_at'))
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/base_relation_factory.rb b/lib/gitlab/import_export/base_relation_factory.rb
deleted file mode 100644
index fcb516fb3a1..00000000000
--- a/lib/gitlab/import_export/base_relation_factory.rb
+++ /dev/null
@@ -1,306 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class BaseRelationFactory
- include Gitlab::Utils::StrongMemoize
-
- IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
-
- OVERRIDES = {}.freeze
- EXISTING_OBJECT_RELATIONS = %i[].freeze
-
- # This represents all relations that have unique key on `project_id` or `group_id`
- UNIQUE_RELATIONS = %i[].freeze
-
- USER_REFERENCES = %w[
- author_id
- assignee_id
- updated_by_id
- merged_by_id
- latest_closed_by_id
- user_id
- created_by_id
- last_edited_by_id
- merge_user_id
- resolved_by_id
- closed_by_id
- owner_id
- ].freeze
-
- TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
-
- def self.create(*args)
- new(*args).create
- end
-
- def self.relation_class(relation_name)
- # There are scenarios where the model is pluralized (e.g.
- # MergeRequest::Metrics), and we don't want to force it to singular
- # with #classify.
- relation_name.to_s.classify.constantize
- rescue NameError
- relation_name.to_s.constantize
- end
-
- def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: [])
- @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
- @relation_hash = relation_hash.except('noteable_id')
- @members_mapper = members_mapper
- @object_builder = object_builder
- @user = user
- @importable = importable
- @imported_object_retries = 0
- @relation_hash[importable_column_name] = @importable.id
-
- # Remove excluded keys from relation_hash
- # We don't do this in the parsed_relation_hash because of the 'transformed attributes'
- # For example, MergeRequestDiffFiles exports its diff attribute as utf8_diff. Then,
- # in the create method that attribute is renamed to diff. And because diff is an excluded key,
- # if we clean the excluded keys in the parsed_relation_hash, it will be removed
- # from the object attributes and the export will fail.
- @relation_hash.except!(*excluded_keys)
- end
-
- # Creates an object from an actual model with name "relation_sym" with params from
- # the relation_hash, updating references with new object IDs, mapping users using
- # the "members_mapper" object, also updating notes if required.
- def create
- return if invalid_relation?
-
- setup_base_models
- setup_models
-
- generate_imported_object
- end
-
- def self.overrides
- self::OVERRIDES
- end
-
- def self.existing_object_relations
- self::EXISTING_OBJECT_RELATIONS
- end
-
- private
-
- def invalid_relation?
- false
- end
-
- def setup_models
- raise NotImplementedError
- end
-
- def unique_relations
- # define in sub-class if any
- self.class::UNIQUE_RELATIONS
- end
-
- def setup_base_models
- update_user_references
- remove_duplicate_assignees
- reset_tokens!
- remove_encrypted_attributes!
- end
-
- def update_user_references
- self.class::USER_REFERENCES.each do |reference|
- if @relation_hash[reference]
- @relation_hash[reference] = @members_mapper.map[@relation_hash[reference]]
- end
- end
- end
-
- def remove_duplicate_assignees
- return unless @relation_hash['issue_assignees']
-
- # When an assignee did not exist in the members mapper, the importer is
- # assigned. We only need to assign each user once.
- @relation_hash['issue_assignees'].uniq!(&:user_id)
- end
-
- def generate_imported_object
- imported_object
- end
-
- def reset_tokens!
- return unless Gitlab::ImportExport.reset_tokens? && self.class::TOKEN_RESET_MODELS.include?(@relation_name)
-
- # If we import/export to the same instance, tokens will have to be reset.
- # We also have to reset them to avoid issues when the gitlab secrets file cannot be copied across.
- relation_class.attribute_names.select { |name| name.include?('token') }.each do |token|
- @relation_hash[token] = nil
- end
- end
-
- def remove_encrypted_attributes!
- return unless relation_class.respond_to?(:encrypted_attributes) && relation_class.encrypted_attributes.any?
-
- relation_class.encrypted_attributes.each_key do |key|
- @relation_hash[key.to_s] = nil
- end
- end
-
- def relation_class
- @relation_class ||= self.class.relation_class(@relation_name)
- end
-
- def importable_column_name
- importable_class_name.concat('_id')
- end
-
- def importable_class_name
- @importable.class.to_s.downcase
- end
-
- def imported_object
- if existing_or_new_object.respond_to?(:importing)
- existing_or_new_object.importing = true
- end
-
- existing_or_new_object
- rescue ActiveRecord::RecordNotUnique
- # as the operation is not atomic, retry in the unlikely scenario an INSERT is
- # performed on the same object between the SELECT and the INSERT
- @imported_object_retries += 1
- retry if @imported_object_retries < IMPORTED_OBJECT_MAX_RETRIES
- end
-
- def parsed_relation_hash
- @parsed_relation_hash ||= Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: @relation_hash,
- relation_class: relation_class)
- end
-
- def existing_or_new_object
- # Only find existing records to avoid mapping tables such as milestones
- # Otherwise always create the record, skipping the extra SELECT clause.
- @existing_or_new_object ||= begin
- if existing_object?
- attribute_hash = attribute_hash_for(['events'])
-
- existing_object.assign_attributes(attribute_hash) if attribute_hash.any?
-
- existing_object
- else
- # Because of single-type inheritance, we need to be careful to use the `type` field
- # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497
- inheritance_column = relation_class.try(:inheritance_column)
- inheritance_attributes = parsed_relation_hash.slice(inheritance_column)
- object = relation_class.new(inheritance_attributes)
- object.assign_attributes(parsed_relation_hash)
- object
- end
- end
- end
-
- def attribute_hash_for(attributes)
- attributes.each_with_object({}) do |hash, value|
- hash[value] = parsed_relation_hash.delete(value) if parsed_relation_hash[value]
- hash
- end
- end
-
- def existing_object
- @existing_object ||= find_or_create_object!
- end
-
- def unique_relation_object
- unique_relation_object = relation_class.find_or_create_by(importable_column_name => @importable.id)
- unique_relation_object.assign_attributes(parsed_relation_hash)
- unique_relation_object
- end
-
- def find_or_create_object!
- return unique_relation_object if unique_relation?
-
- # Can't use IDs as validation exists calling `group` or `project` attributes
- finder_hash = parsed_relation_hash.tap do |hash|
- if relation_class.attribute_method?('group_id') && @importable.is_a?(Project)
- hash['group'] = @importable.group
- end
-
- hash[importable_class_name] = @importable if relation_class.reflect_on_association(importable_class_name.to_sym)
- hash.delete(importable_column_name)
- end
-
- @object_builder.build(relation_class, finder_hash)
- end
-
- def setup_note
- set_note_author
- # attachment is deprecated and note uploads are handled by Markdown uploader
- @relation_hash['attachment'] = nil
- end
-
- # Sets the author for a note. If the user importing the project
- # has admin access, an actual mapping with new project members
- # will be used. Otherwise, a note stating the original author name
- # is left.
- def set_note_author
- old_author_id = @relation_hash['author_id']
- author = @relation_hash.delete('author')
-
- update_note_for_missing_author(author['name']) unless has_author?(old_author_id)
- end
-
- def has_author?(old_author_id)
- admin_user? && @members_mapper.include?(old_author_id)
- end
-
- def missing_author_note(updated_at, author_name)
- timestamp = updated_at.split('.').first
- "\n\n *By #{author_name} on #{timestamp} (imported from GitLab project)*"
- end
-
- def update_note_for_missing_author(author_name)
- @relation_hash['note'] = '*Blank note*' if @relation_hash['note'].blank?
- @relation_hash['note'] = "#{@relation_hash['note']}#{missing_author_note(@relation_hash['updated_at'], author_name)}"
- end
-
- def admin_user?
- @user.admin?
- end
-
- def existing_object?
- strong_memoize(:_existing_object) do
- self.class.existing_object_relations.include?(@relation_name) || unique_relation?
- end
- end
-
- def unique_relation?
- strong_memoize(:unique_relation) do
- importable_foreign_key.present? &&
- (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?)
- end
- end
-
- def has_unique_index_on_importable_fk?
- cache = cached_has_unique_index_on_importable_fk
- table_name = relation_class.table_name
- return cache[table_name] if cache.has_key?(table_name)
-
- index_exists =
- ActiveRecord::Base.connection.index_exists?(
- relation_class.table_name,
- importable_foreign_key,
- unique: true)
-
- cache[table_name] = index_exists
- end
-
- # Avoid unnecessary DB requests
- def cached_has_unique_index_on_importable_fk
- Thread.current[:cached_has_unique_index_on_importable_fk] ||= {}
- end
-
- def uses_importable_fk_as_primary_key?
- relation_class.primary_key == importable_foreign_key
- end
-
- def importable_foreign_key
- relation_class.reflect_on_association(importable_class_name.to_sym)&.foreign_key
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/group_import_export.yml b/lib/gitlab/import_export/group/import_export.yml
index d4e0ff12373..d4e0ff12373 100644
--- a/lib/gitlab/import_export/group_import_export.yml
+++ b/lib/gitlab/import_export/group/import_export.yml
diff --git a/lib/gitlab/import_export/group/object_builder.rb b/lib/gitlab/import_export/group/object_builder.rb
new file mode 100644
index 00000000000..e171a31348e
--- /dev/null
+++ b/lib/gitlab/import_export/group/object_builder.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ # Given a class, it finds or creates a new object at group level.
+ #
+ # Example:
+ # `Group::ObjectBuilder.build(Label, label_attributes)`
+ # finds or initializes a label with the given attributes.
+ class ObjectBuilder < Base::ObjectBuilder
+ def self.build(*args)
+ ::Group.transaction do
+ super
+ end
+ end
+
+ def initialize(klass, attributes)
+ super
+
+ @group = @attributes['group']
+
+ update_description
+ end
+
+ private
+
+ attr_reader :group
+
+ # Convert description empty string to nil
+ # due to existing object being saved with description: nil
+ # Which makes object lookup to fail since nil != ''
+ def update_description
+ attributes['description'] = nil if attributes['description'] == ''
+ end
+
+ def where_clauses
+ [
+ where_clause_base,
+ where_clause_for_title,
+ where_clause_for_description,
+ where_clause_for_created_at
+ ].compact
+ end
+
+ # Returns Arel clause `"{table_name}"."group_id" = {group.id}`
+ def where_clause_base
+ table[:group_id].in(group_and_ancestor_ids)
+ end
+
+ def group_and_ancestor_ids
+ group.ancestors.map(&:id) << group.id
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb
new file mode 100644
index 00000000000..91637161377
--- /dev/null
+++ b/lib/gitlab/import_export/group/relation_factory.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class RelationFactory < Base::RelationFactory
+ OVERRIDES = {
+ labels: :group_labels,
+ priorities: :label_priorities,
+ label: :group_label,
+ parent: :epic
+ }.freeze
+
+ EXISTING_OBJECT_RELATIONS = %i[
+ epic
+ epics
+ milestone
+ milestones
+ label
+ labels
+ group_label
+ group_labels
+ ].freeze
+
+ private
+
+ def setup_models
+ setup_note if @relation_name == :notes
+
+ update_group_references
+ end
+
+ def update_group_references
+ return unless self.class.existing_object_relations.include?(@relation_name)
+ return unless @relation_hash['group_id']
+
+ @relation_hash['group_id'] = @importable.id
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
new file mode 100644
index 00000000000..e6f49dcac7a
--- /dev/null
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class TreeRestorer
+ attr_reader :user
+ attr_reader :shared
+ attr_reader :group
+
+ def initialize(user:, shared:, group:, group_hash:)
+ @path = File.join(shared.export_path, 'group.json')
+ @user = user
+ @shared = shared
+ @group = group
+ @group_hash = group_hash
+ end
+
+ def restore
+ @tree_hash = @group_hash || read_tree_hash
+ @group_members = @tree_hash.delete('members')
+ @children = @tree_hash.delete('children')
+
+ if members_mapper.map && restorer.restore
+ @children&.each do |group_hash|
+ group = create_group(group_hash: group_hash, parent_group: @group)
+ shared = Gitlab::ImportExport::Shared.new(group)
+
+ self.class.new(
+ user: @user,
+ shared: shared,
+ group: group,
+ group_hash: group_hash
+ ).restore
+ end
+ end
+
+ return false if @shared.errors.any?
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ def read_tree_hash
+ json = IO.read(@path)
+ ActiveSupport::JSON.decode(json)
+ rescue => e
+ @shared.logger.error(
+ group_id: @group.id,
+ group_name: @group.name,
+ message: "Import/Export error: #{e.message}"
+ )
+
+ raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+ end
+
+ def restorer
+ @relation_tree_restorer ||= RelationTreeRestorer.new(
+ user: @user,
+ shared: @shared,
+ importable: @group,
+ tree_hash: @tree_hash.except('name', 'path'),
+ members_mapper: members_mapper,
+ object_builder: object_builder,
+ relation_factory: relation_factory,
+ reader: reader
+ )
+ end
+
+ def create_group(group_hash:, parent_group:)
+ group_params = {
+ name: group_hash['name'],
+ path: group_hash['path'],
+ parent_id: parent_group&.id,
+ visibility_level: sub_group_visibility_level(group_hash, parent_group)
+ }
+
+ ::Groups::CreateService.new(@user, group_params).execute
+ end
+
+ def sub_group_visibility_level(group_hash, parent_group)
+ original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
+
+ if parent_group && parent_group.visibility_level < original_visibility_level
+ Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+ else
+ original_visibility_level
+ end
+ end
+
+ def members_mapper
+ @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
+ end
+
+ def relation_factory
+ Gitlab::ImportExport::Group::RelationFactory
+ end
+
+ def object_builder
+ Gitlab::ImportExport::Group::ObjectBuilder
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ )
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
new file mode 100644
index 00000000000..48f6925884b
--- /dev/null
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Group
+ class TreeSaver
+ attr_reader :full_path, :shared
+
+ def initialize(group:, current_user:, shared:, params: {})
+ @params = params
+ @current_user = current_user
+ @shared = shared
+ @group = group
+ @full_path = File.join(@shared.export_path, ImportExport.group_filename)
+ end
+
+ def save
+ group_tree = serialize(@group, reader.group_tree)
+ tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ def serialize(group, relations_tree)
+ group_tree = tree_saver.serialize(group, relations_tree)
+
+ group.children.each do |child|
+ group_tree['children'] ||= []
+ group_tree['children'] << serialize(child, relations_tree)
+ end
+
+ group_tree
+ rescue => e
+ @shared.error(e)
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(
+ shared: @shared,
+ config: Gitlab::ImportExport::Config.new(
+ config: Gitlab::ImportExport.group_config_file
+ ).to_h
+ )
+ end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/group_object_builder.rb b/lib/gitlab/import_export/group_object_builder.rb
deleted file mode 100644
index 9796bfa07d4..00000000000
--- a/lib/gitlab/import_export/group_object_builder.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- # Given a class, it finds or creates a new object at group level.
- #
- # Example:
- # `GroupObjectBuilder.build(Label, label_attributes)`
- # finds or initializes a label with the given attributes.
- class GroupObjectBuilder < BaseObjectBuilder
- def self.build(*args)
- Group.transaction do
- super
- end
- end
-
- def initialize(klass, attributes)
- super
-
- @group = @attributes['group']
-
- update_description
- end
-
- private
-
- attr_reader :group
-
- # Convert description empty string to nil
- # due to existing object being saved with description: nil
- # Which makes object lookup to fail since nil != ''
- def update_description
- attributes['description'] = nil if attributes['description'] == ''
- end
-
- def where_clauses
- [
- where_clause_base,
- where_clause_for_title,
- where_clause_for_description,
- where_clause_for_created_at
- ].compact
- end
-
- # Returns Arel clause `"{table_name}"."group_id" = {group.id}`
- def where_clause_base
- table[:group_id].in(group_and_ancestor_ids)
- end
-
- def group_and_ancestor_ids
- group.ancestors.map(&:id) << group.id
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/group_project_object_builder.rb b/lib/gitlab/import_export/group_project_object_builder.rb
deleted file mode 100644
index 9e8f9d11393..00000000000
--- a/lib/gitlab/import_export/group_project_object_builder.rb
+++ /dev/null
@@ -1,117 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- # Given a class, it finds or creates a new object
- # (initializes in the case of Label) at group or project level.
- # If it does not exist in the group, it creates it at project level.
- #
- # Example:
- # `GroupProjectObjectBuilder.build(Label, label_attributes)`
- # finds or initializes a label with the given attributes.
- #
- # It also adds some logic around Group Labels/Milestones for edge cases.
- class GroupProjectObjectBuilder < BaseObjectBuilder
- def self.build(*args)
- Project.transaction do
- super
- end
- end
-
- def initialize(klass, attributes)
- super
-
- @group = @attributes['group']
- @project = @attributes['project']
- end
-
- def find
- return if epic? && group.nil?
-
- super
- end
-
- private
-
- attr_reader :group, :project
-
- def where_clauses
- [
- where_clause_base,
- where_clause_for_title,
- where_clause_for_klass
- ].compact
- end
-
- # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
- # For example: merge_request has :target_project_id, and we are searching by :iid
- # or, if group is present:
- # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
- def where_clause_base
- [].tap do |clauses|
- clauses << table[:project_id].eq(project.id) if project
- clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
- end.reduce(:or)
- end
-
- # Returns Arel clause for a particular model or `nil`.
- def where_clause_for_klass
- attrs_to_arel(attributes.slice('iid')) if merge_request?
- end
-
- def prepare_attributes
- attributes.dup.tap do |atts|
- atts.delete('group') unless epic?
-
- if label?
- atts['type'] = 'ProjectLabel' # Always create project labels
- elsif milestone?
- if atts['group_id'] # Transform new group milestones into project ones
- atts['iid'] = nil
- atts.delete('group_id')
- else
- claim_iid
- end
- end
-
- atts['importing'] = true if klass.ancestors.include?(Importable)
- end
- end
-
- def label?
- klass == Label
- end
-
- def milestone?
- klass == Milestone
- end
-
- def merge_request?
- klass == MergeRequest
- end
-
- def epic?
- klass == Epic
- end
-
- # If an existing group milestone used the IID
- # claim the IID back and set the group milestone to use one available
- # This is necessary to fix situations like the following:
- # - Importing into a user namespace project with exported group milestones
- # where the IID of the Group milestone could conflict with a project one.
- def claim_iid
- # The milestone has to be a group milestone, as it's the only case where
- # we set the IID as the maximum. The rest of them are fixed.
- milestone = project.milestones.find_by(iid: attributes['iid'])
-
- return unless milestone
-
- milestone.iid = nil
- milestone.ensure_project_iid!
- milestone.save!
- end
- end
- end
-end
-
-Gitlab::ImportExport::GroupProjectObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::GroupProjectObjectBuilder')
diff --git a/lib/gitlab/import_export/group_relation_factory.rb b/lib/gitlab/import_export/group_relation_factory.rb
deleted file mode 100644
index e3597af44d2..00000000000
--- a/lib/gitlab/import_export/group_relation_factory.rb
+++ /dev/null
@@ -1,40 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class GroupRelationFactory < BaseRelationFactory
- OVERRIDES = {
- labels: :group_labels,
- priorities: :label_priorities,
- label: :group_label,
- parent: :epic
- }.freeze
-
- EXISTING_OBJECT_RELATIONS = %i[
- epic
- epics
- milestone
- milestones
- label
- labels
- group_label
- group_labels
- ].freeze
-
- private
-
- def setup_models
- setup_note if @relation_name == :notes
-
- update_group_references
- end
-
- def update_group_references
- return unless self.class.existing_object_relations.include?(@relation_name)
- return unless @relation_hash['group_id']
-
- @relation_hash['group_id'] = @importable.id
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/group_tree_restorer.rb b/lib/gitlab/import_export/group_tree_restorer.rb
deleted file mode 100644
index 2f42843ed6c..00000000000
--- a/lib/gitlab/import_export/group_tree_restorer.rb
+++ /dev/null
@@ -1,116 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class GroupTreeRestorer
- attr_reader :user
- attr_reader :shared
- attr_reader :group
-
- def initialize(user:, shared:, group:, group_hash:)
- @path = File.join(shared.export_path, 'group.json')
- @user = user
- @shared = shared
- @group = group
- @group_hash = group_hash
- end
-
- def restore
- @tree_hash = @group_hash || read_tree_hash
- @group_members = @tree_hash.delete('members')
- @children = @tree_hash.delete('children')
-
- if members_mapper.map && restorer.restore
- @children&.each do |group_hash|
- group = create_group(group_hash: group_hash, parent_group: @group)
- shared = Gitlab::ImportExport::Shared.new(group)
-
- self.class.new(
- user: @user,
- shared: shared,
- group: group,
- group_hash: group_hash
- ).restore
- end
- end
-
- return false if @shared.errors.any?
-
- true
- rescue => e
- @shared.error(e)
- false
- end
-
- private
-
- def read_tree_hash
- json = IO.read(@path)
- ActiveSupport::JSON.decode(json)
- rescue => e
- @shared.logger.error(
- group_id: @group.id,
- group_name: @group.name,
- message: "Import/Export error: #{e.message}"
- )
-
- raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
- end
-
- def restorer
- @relation_tree_restorer ||= RelationTreeRestorer.new(
- user: @user,
- shared: @shared,
- importable: @group,
- tree_hash: @tree_hash.except('name', 'path'),
- members_mapper: members_mapper,
- object_builder: object_builder,
- relation_factory: relation_factory,
- reader: reader
- )
- end
-
- def create_group(group_hash:, parent_group:)
- group_params = {
- name: group_hash['name'],
- path: group_hash['path'],
- parent_id: parent_group&.id,
- visibility_level: sub_group_visibility_level(group_hash, parent_group)
- }
-
- ::Groups::CreateService.new(@user, group_params).execute
- end
-
- def sub_group_visibility_level(group_hash, parent_group)
- original_visibility_level = group_hash['visibility_level'] || Gitlab::VisibilityLevel::PRIVATE
-
- if parent_group && parent_group.visibility_level < original_visibility_level
- Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
- else
- original_visibility_level
- end
- end
-
- def members_mapper
- @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @group_members, user: @user, importable: @group)
- end
-
- def relation_factory
- Gitlab::ImportExport::GroupRelationFactory
- end
-
- def object_builder
- Gitlab::ImportExport::GroupObjectBuilder
- end
-
- def reader
- @reader ||= Gitlab::ImportExport::Reader.new(
- shared: @shared,
- config: Gitlab::ImportExport::Config.new(
- config: Gitlab::ImportExport.group_config_file
- ).to_h
- )
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/group_tree_saver.rb b/lib/gitlab/import_export/group_tree_saver.rb
deleted file mode 100644
index 2effcd01e30..00000000000
--- a/lib/gitlab/import_export/group_tree_saver.rb
+++ /dev/null
@@ -1,55 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class GroupTreeSaver
- attr_reader :full_path, :shared
-
- def initialize(group:, current_user:, shared:, params: {})
- @params = params
- @current_user = current_user
- @shared = shared
- @group = group
- @full_path = File.join(@shared.export_path, ImportExport.group_filename)
- end
-
- def save
- group_tree = serialize(@group, reader.group_tree)
- tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
-
- true
- rescue => e
- @shared.error(e)
- false
- end
-
- private
-
- def serialize(group, relations_tree)
- group_tree = tree_saver.serialize(group, relations_tree)
-
- group.children.each do |child|
- group_tree['children'] ||= []
- group_tree['children'] << serialize(child, relations_tree)
- end
-
- group_tree
- rescue => e
- @shared.error(e)
- end
-
- def reader
- @reader ||= Gitlab::ImportExport::Reader.new(
- shared: @shared,
- config: Gitlab::ImportExport::Config.new(
- config: Gitlab::ImportExport.group_config_file
- ).to_h
- )
- end
-
- def tree_saver
- @tree_saver ||= RelationTreeSaver.new
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index a6463ed678c..4eeecc14067 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -49,7 +49,7 @@ module Gitlab
end
def project_tree
- @project_tree ||= Gitlab::ImportExport::ProjectTreeRestorer.new(user: current_user,
+ @project_tree ||= Gitlab::ImportExport::Project::TreeRestorer.new(user: current_user,
shared: shared,
project: project)
end
@@ -125,7 +125,7 @@ module Gitlab
def project_to_overwrite
strong_memoize(:project_to_overwrite) do
- Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
+ ::Project.find_by_full_path("#{project.namespace.full_path}/#{original_path}")
end
end
end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index e7eae0a8c31..fd76252eb36 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -91,9 +91,9 @@ module Gitlab
def relation_class
case @importable
- when Project
+ when ::Project
ProjectMember
- when Group
+ when ::Group
GroupMember
end
end
diff --git a/lib/gitlab/import_export/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 4fa909ac94b..4fa909ac94b 100644
--- a/lib/gitlab/import_export/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
new file mode 100644
index 00000000000..c3637b1c115
--- /dev/null
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ # Given a class, it finds or creates a new object
+ # (initializes in the case of Label) at group or project level.
+ # If it does not exist in the group, it creates it at project level.
+ #
+ # Example:
+ # `ObjectBuilder.build(Label, label_attributes)`
+ # finds or initializes a label with the given attributes.
+ #
+ # It also adds some logic around Group Labels/Milestones for edge cases.
+ class ObjectBuilder < Base::ObjectBuilder
+ def self.build(*args)
+ ::Project.transaction do
+ super
+ end
+ end
+
+ def initialize(klass, attributes)
+ super
+
+ @group = @attributes['group']
+ @project = @attributes['project']
+ end
+
+ def find
+ return if epic? && group.nil?
+
+ super
+ end
+
+ private
+
+ attr_reader :group, :project
+
+ def where_clauses
+ [
+ where_clause_base,
+ where_clause_for_title,
+ where_clause_for_klass
+ ].compact
+ end
+
+ # Returns Arel clause `"{table_name}"."project_id" = {project.id}` if project is present
+ # For example: merge_request has :target_project_id, and we are searching by :iid
+ # or, if group is present:
+ # `"{table_name}"."project_id" = {project.id} OR "{table_name}"."group_id" = {group.id}`
+ def where_clause_base
+ [].tap do |clauses|
+ clauses << table[:project_id].eq(project.id) if project
+ clauses << table[:group_id].in(group.self_and_ancestors_ids) if group
+ end.reduce(:or)
+ end
+
+ # Returns Arel clause for a particular model or `nil`.
+ def where_clause_for_klass
+ attrs_to_arel(attributes.slice('iid')) if merge_request?
+ end
+
+ def prepare_attributes
+ attributes.dup.tap do |atts|
+ atts.delete('group') unless epic?
+
+ if label?
+ atts['type'] = 'ProjectLabel' # Always create project labels
+ elsif milestone?
+ if atts['group_id'] # Transform new group milestones into project ones
+ atts['iid'] = nil
+ atts.delete('group_id')
+ else
+ claim_iid
+ end
+ end
+
+ atts['importing'] = true if klass.ancestors.include?(Importable)
+ end
+ end
+
+ def label?
+ klass == Label
+ end
+
+ def milestone?
+ klass == Milestone
+ end
+
+ def merge_request?
+ klass == MergeRequest
+ end
+
+ def epic?
+ klass == Epic
+ end
+
+ # If an existing group milestone used the IID
+ # claim the IID back and set the group milestone to use one available
+ # This is necessary to fix situations like the following:
+ # - Importing into a user namespace project with exported group milestones
+ # where the IID of the Group milestone could conflict with a project one.
+ def claim_iid
+ # The milestone has to be a group milestone, as it's the only case where
+ # we set the IID as the maximum. The rest of them are fixed.
+ milestone = project.milestones.find_by(iid: attributes['iid'])
+
+ return unless milestone
+
+ milestone.iid = nil
+ milestone.ensure_project_iid!
+ milestone.save!
+ end
+ end
+ end
+ end
+end
+
+Gitlab::ImportExport::Project::ObjectBuilder.prepend_if_ee('EE::Gitlab::ImportExport::Project::ObjectBuilder')
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
new file mode 100644
index 00000000000..951482a933a
--- /dev/null
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -0,0 +1,162 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ class RelationFactory < Base::RelationFactory
+ prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
+
+ OVERRIDES = { snippets: :project_snippets,
+ ci_pipelines: 'Ci::Pipeline',
+ pipelines: 'Ci::Pipeline',
+ stages: 'Ci::Stage',
+ statuses: 'commit_status',
+ triggers: 'Ci::Trigger',
+ pipeline_schedules: 'Ci::PipelineSchedule',
+ builds: 'Ci::Build',
+ runners: 'Ci::Runner',
+ hooks: 'ProjectHook',
+ merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
+ push_access_levels: 'ProtectedBranch::PushAccessLevel',
+ create_access_levels: 'ProtectedTag::CreateAccessLevel',
+ labels: :project_labels,
+ priorities: :label_priorities,
+ auto_devops: :project_auto_devops,
+ label: :project_label,
+ custom_attributes: 'ProjectCustomAttribute',
+ project_badges: 'Badge',
+ metrics: 'MergeRequest::Metrics',
+ ci_cd_settings: 'ProjectCiCdSetting',
+ error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
+ links: 'Releases::Link',
+ metrics_setting: 'ProjectMetricsSetting' }.freeze
+
+ BUILD_MODELS = %i[Ci::Build commit_status].freeze
+
+ GROUP_REFERENCES = %w[group_id].freeze
+
+ PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
+
+ EXISTING_OBJECT_RELATIONS = %i[
+ milestone
+ milestones
+ label
+ labels
+ project_label
+ project_labels
+ group_label
+ group_labels
+ project_feature
+ merge_request
+ epic
+ ProjectCiCdSetting
+ container_expiration_policy
+ ].freeze
+
+ def create
+ @object = super
+
+ # We preload the project, user, and group to re-use objects
+ @object = preload_keys(@object, PROJECT_REFERENCES, @importable)
+ @object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
+ @object = preload_keys(@object, USER_REFERENCES, @user)
+ end
+
+ private
+
+ def invalid_relation?
+ # Do not create relation if it is:
+ # - An unknown service
+ # - A legacy trigger
+ unknown_service? ||
+ (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
+ end
+
+ def setup_models
+ case @relation_name
+ when :merge_request_diff_files then setup_diff
+ when :notes then setup_note
+ when :'Ci::Pipeline' then setup_pipeline
+ when *BUILD_MODELS then setup_build
+ end
+
+ update_project_references
+ update_group_references
+ end
+
+ def generate_imported_object
+ if @relation_name == :merge_requests
+ MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
+ else
+ super
+ end
+ end
+
+ def update_project_references
+ # If source and target are the same, populate them with the new project ID.
+ if @relation_hash['source_project_id']
+ @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
+ end
+
+ @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
+ end
+
+ def same_source_and_target?
+ @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
+ end
+
+ def update_group_references
+ return unless existing_object?
+ return unless @relation_hash['group_id']
+
+ @relation_hash['group_id'] = @importable.namespace_id
+ end
+
+ def setup_build
+ @relation_hash.delete('trace') # old export files have trace
+ @relation_hash.delete('token')
+ @relation_hash.delete('commands')
+ @relation_hash.delete('artifacts_file_store')
+ @relation_hash.delete('artifacts_metadata_store')
+ @relation_hash.delete('artifacts_size')
+ end
+
+ def setup_diff
+ @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
+ end
+
+ def setup_pipeline
+ @relation_hash.fetch('stages', []).each do |stage|
+ stage.statuses.each do |status|
+ status.pipeline = imported_object
+ end
+ end
+ end
+
+ def unknown_service?
+ @relation_name == :services && parsed_relation_hash['type'] &&
+ !Object.const_defined?(parsed_relation_hash['type'])
+ end
+
+ def legacy_trigger?
+ @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
+ end
+
+ def preload_keys(object, references, value)
+ return object unless value
+
+ references.each do |key|
+ attribute = "#{key.delete_suffix('_id')}=".to_sym
+ next unless object.respond_to?(key) && object.respond_to?(attribute)
+
+ if object.read_attribute(key) == value&.id
+ object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+
+ object
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_loader.rb b/lib/gitlab/import_export/project/tree_loader.rb
new file mode 100644
index 00000000000..6d4737a2d00
--- /dev/null
+++ b/lib/gitlab/import_export/project/tree_loader.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ class TreeLoader
+ def load(path, dedup_entries: false)
+ tree_hash = ActiveSupport::JSON.decode(IO.read(path))
+
+ if dedup_entries
+ dedup_tree(tree_hash)
+ else
+ tree_hash
+ end
+ end
+
+ private
+
+ # This function removes duplicate entries from the given tree recursively
+ # by caching nodes it encounters repeatedly. We only consider nodes for
+ # which there can actually be multiple equivalent instances (e.g. strings,
+ # hashes and arrays, but not `nil`s, numbers or booleans.)
+ #
+ # The algorithm uses a recursive depth-first descent with 3 cases, starting
+ # with a root node (the tree/hash itself):
+ # - a node has already been cached; in this case we return it from the cache
+ # - a node has not been cached yet but should be; descend into its children
+ # - a node is neither cached nor qualifies for caching; this is a no-op
+ def dedup_tree(node, nodes_seen = {})
+ if nodes_seen.key?(node) && distinguishable?(node)
+ yield nodes_seen[node]
+ elsif should_dedup?(node)
+ nodes_seen[node] = node
+
+ case node
+ when Array
+ node.each_index do |idx|
+ dedup_tree(node[idx], nodes_seen) do |cached_node|
+ node[idx] = cached_node
+ end
+ end
+ when Hash
+ node.each do |k, v|
+ dedup_tree(v, nodes_seen) do |cached_node|
+ node[k] = cached_node
+ end
+ end
+ end
+ else
+ node
+ end
+ end
+
+ # We do not need to consider nodes for which there cannot be multiple instances
+ def should_dedup?(node)
+ node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
+ end
+
+ # We can only safely de-dup values that are distinguishable. True value objects
+ # are always distinguishable by nature. Hashes however can represent entities,
+ # which are identified by ID, not value. We therefore disallow de-duping hashes
+ # that do not have an `id` field, since we might risk dropping entities that
+ # have equal attributes yet different identities.
+ def distinguishable?(node)
+ if node.is_a?(Hash)
+ node.key?('id')
+ else
+ true
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
new file mode 100644
index 00000000000..a5123f16dbc
--- /dev/null
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ class TreeRestorer
+ LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
+
+ attr_reader :user
+ attr_reader :shared
+ attr_reader :project
+
+ def initialize(user:, shared:, project:)
+ @user = user
+ @shared = shared
+ @project = project
+ @tree_loader = TreeLoader.new
+ end
+
+ def restore
+ @tree_hash = read_tree_hash
+ @project_members = @tree_hash.delete('project_members')
+
+ RelationRenameService.rename(@tree_hash)
+
+ if relation_tree_restorer.restore
+ import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
+ @project.merge_requests.set_latest_merge_request_diff_ids!
+ end
+
+ true
+ else
+ false
+ end
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ def large_project?(path)
+ File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
+ end
+
+ def read_tree_hash
+ path = File.join(@shared.export_path, 'project.json')
+ dedup_entries = large_project?(path) &&
+ Feature.enabled?(:dedup_project_import_metadata, project.group)
+
+ @tree_loader.load(path, dedup_entries: dedup_entries)
+ rescue => e
+ Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
+ raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+ end
+
+ def relation_tree_restorer
+ @relation_tree_restorer ||= RelationTreeRestorer.new(
+ user: @user,
+ shared: @shared,
+ importable: @project,
+ tree_hash: @tree_hash,
+ object_builder: object_builder,
+ members_mapper: members_mapper,
+ relation_factory: relation_factory,
+ reader: reader
+ )
+ end
+
+ def members_mapper
+ @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
+ user: @user,
+ importable: @project)
+ end
+
+ def object_builder
+ Project::ObjectBuilder
+ end
+
+ def relation_factory
+ Project::RelationFactory
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+ end
+
+ def import_failure_service
+ @import_failure_service ||= ImportFailureService.new(@project)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
new file mode 100644
index 00000000000..58f33a04851
--- /dev/null
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module ImportExport
+ module Project
+ class TreeSaver
+ attr_reader :full_path
+
+ def initialize(project:, current_user:, shared:, params: {})
+ @params = params
+ @project = project
+ @current_user = current_user
+ @shared = shared
+ @full_path = File.join(@shared.export_path, ImportExport.project_filename)
+ end
+
+ def save
+ project_tree = tree_saver.serialize(@project, reader.project_tree)
+ fix_project_tree(project_tree)
+ tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
+
+ true
+ rescue => e
+ @shared.error(e)
+ false
+ end
+
+ private
+
+ # Aware that the resulting hash needs to be pure-hash and
+ # does not include any AR objects anymore, only objects that run `.to_json`
+ def fix_project_tree(project_tree)
+ if @params[:description].present?
+ project_tree['description'] = @params[:description]
+ end
+
+ project_tree['project_members'] += group_members_array
+
+ RelationRenameService.add_new_associations(project_tree)
+ end
+
+ def reader
+ @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
+ end
+
+ def group_members_array
+ group_members.as_json(reader.group_members_tree).each do |group_member|
+ group_member['source_type'] = 'Project' # Make group members project members of the future import
+ end
+ end
+
+ def group_members
+ return [] unless @current_user.can?(:admin_group, @project.group)
+
+ # We need `.where.not(user_id: nil)` here otherwise when a group has an
+ # invitee, it would make the following query return 0 rows since a NULL
+ # user_id would be present in the subquery
+ # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
+ non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
+
+ GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
+ end
+
+ def tree_saver
+ @tree_saver ||= RelationTreeSaver.new
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project_relation_factory.rb b/lib/gitlab/import_export/project_relation_factory.rb
deleted file mode 100644
index 0e08a66b89c..00000000000
--- a/lib/gitlab/import_export/project_relation_factory.rb
+++ /dev/null
@@ -1,160 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class ProjectRelationFactory < BaseRelationFactory
- prepend_if_ee('::EE::Gitlab::ImportExport::ProjectRelationFactory') # rubocop: disable Cop/InjectEnterpriseEditionModule
-
- OVERRIDES = { snippets: :project_snippets,
- ci_pipelines: 'Ci::Pipeline',
- pipelines: 'Ci::Pipeline',
- stages: 'Ci::Stage',
- statuses: 'commit_status',
- triggers: 'Ci::Trigger',
- pipeline_schedules: 'Ci::PipelineSchedule',
- builds: 'Ci::Build',
- runners: 'Ci::Runner',
- hooks: 'ProjectHook',
- merge_access_levels: 'ProtectedBranch::MergeAccessLevel',
- push_access_levels: 'ProtectedBranch::PushAccessLevel',
- create_access_levels: 'ProtectedTag::CreateAccessLevel',
- labels: :project_labels,
- priorities: :label_priorities,
- auto_devops: :project_auto_devops,
- label: :project_label,
- custom_attributes: 'ProjectCustomAttribute',
- project_badges: 'Badge',
- metrics: 'MergeRequest::Metrics',
- ci_cd_settings: 'ProjectCiCdSetting',
- error_tracking_setting: 'ErrorTracking::ProjectErrorTrackingSetting',
- links: 'Releases::Link',
- metrics_setting: 'ProjectMetricsSetting' }.freeze
-
- BUILD_MODELS = %i[Ci::Build commit_status].freeze
-
- GROUP_REFERENCES = %w[group_id].freeze
-
- PROJECT_REFERENCES = %w[project_id source_project_id target_project_id].freeze
-
- EXISTING_OBJECT_RELATIONS = %i[
- milestone
- milestones
- label
- labels
- project_label
- project_labels
- group_label
- group_labels
- project_feature
- merge_request
- epic
- ProjectCiCdSetting
- container_expiration_policy
- ].freeze
-
- def create
- @object = super
-
- # We preload the project, user, and group to re-use objects
- @object = preload_keys(@object, PROJECT_REFERENCES, @importable)
- @object = preload_keys(@object, GROUP_REFERENCES, @importable.group)
- @object = preload_keys(@object, USER_REFERENCES, @user)
- end
-
- private
-
- def invalid_relation?
- # Do not create relation if it is:
- # - An unknown service
- # - A legacy trigger
- unknown_service? ||
- (!Feature.enabled?(:use_legacy_pipeline_triggers, @importable) && legacy_trigger?)
- end
-
- def setup_models
- case @relation_name
- when :merge_request_diff_files then setup_diff
- when :notes then setup_note
- when :'Ci::Pipeline' then setup_pipeline
- when *BUILD_MODELS then setup_build
- end
-
- update_project_references
- update_group_references
- end
-
- def generate_imported_object
- if @relation_name == :merge_requests
- MergeRequestParser.new(@importable, @relation_hash.delete('diff_head_sha'), super, @relation_hash).parse!
- else
- super
- end
- end
-
- def update_project_references
- # If source and target are the same, populate them with the new project ID.
- if @relation_hash['source_project_id']
- @relation_hash['source_project_id'] = same_source_and_target? ? @relation_hash['project_id'] : MergeRequestParser::FORKED_PROJECT_ID
- end
-
- @relation_hash['target_project_id'] = @relation_hash['project_id'] if @relation_hash['target_project_id']
- end
-
- def same_source_and_target?
- @relation_hash['target_project_id'] && @relation_hash['target_project_id'] == @relation_hash['source_project_id']
- end
-
- def update_group_references
- return unless existing_object?
- return unless @relation_hash['group_id']
-
- @relation_hash['group_id'] = @importable.namespace_id
- end
-
- def setup_build
- @relation_hash.delete('trace') # old export files have trace
- @relation_hash.delete('token')
- @relation_hash.delete('commands')
- @relation_hash.delete('artifacts_file_store')
- @relation_hash.delete('artifacts_metadata_store')
- @relation_hash.delete('artifacts_size')
- end
-
- def setup_diff
- @relation_hash['diff'] = @relation_hash.delete('utf8_diff')
- end
-
- def setup_pipeline
- @relation_hash.fetch('stages', []).each do |stage|
- stage.statuses.each do |status|
- status.pipeline = imported_object
- end
- end
- end
-
- def unknown_service?
- @relation_name == :services && parsed_relation_hash['type'] &&
- !Object.const_defined?(parsed_relation_hash['type'])
- end
-
- def legacy_trigger?
- @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
- end
-
- def preload_keys(object, references, value)
- return object unless value
-
- references.each do |key|
- attribute = "#{key.delete_suffix('_id')}=".to_sym
- next unless object.respond_to?(key) && object.respond_to?(attribute)
-
- if object.read_attribute(key) == value&.id
- object.public_send(attribute, value) # rubocop:disable GitlabSecurity/PublicSend
- end
- end
-
- object
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/project_tree_loader.rb b/lib/gitlab/import_export/project_tree_loader.rb
deleted file mode 100644
index fc21858043d..00000000000
--- a/lib/gitlab/import_export/project_tree_loader.rb
+++ /dev/null
@@ -1,72 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class ProjectTreeLoader
- def load(path, dedup_entries: false)
- tree_hash = ActiveSupport::JSON.decode(IO.read(path))
-
- if dedup_entries
- dedup_tree(tree_hash)
- else
- tree_hash
- end
- end
-
- private
-
- # This function removes duplicate entries from the given tree recursively
- # by caching nodes it encounters repeatedly. We only consider nodes for
- # which there can actually be multiple equivalent instances (e.g. strings,
- # hashes and arrays, but not `nil`s, numbers or booleans.)
- #
- # The algorithm uses a recursive depth-first descent with 3 cases, starting
- # with a root node (the tree/hash itself):
- # - a node has already been cached; in this case we return it from the cache
- # - a node has not been cached yet but should be; descend into its children
- # - a node is neither cached nor qualifies for caching; this is a no-op
- def dedup_tree(node, nodes_seen = {})
- if nodes_seen.key?(node) && distinguishable?(node)
- yield nodes_seen[node]
- elsif should_dedup?(node)
- nodes_seen[node] = node
-
- case node
- when Array
- node.each_index do |idx|
- dedup_tree(node[idx], nodes_seen) do |cached_node|
- node[idx] = cached_node
- end
- end
- when Hash
- node.each do |k, v|
- dedup_tree(v, nodes_seen) do |cached_node|
- node[k] = cached_node
- end
- end
- end
- else
- node
- end
- end
-
- # We do not need to consider nodes for which there cannot be multiple instances
- def should_dedup?(node)
- node && !(node.is_a?(Numeric) || node.is_a?(TrueClass) || node.is_a?(FalseClass))
- end
-
- # We can only safely de-dup values that are distinguishable. True value objects
- # are always distinguishable by nature. Hashes however can represent entities,
- # which are identified by ID, not value. We therefore disallow de-duping hashes
- # that do not have an `id` field, since we might risk dropping entities that
- # have equal attributes yet different identities.
- def distinguishable?(node)
- if node.is_a?(Hash)
- node.key?('id')
- else
- true
- end
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
deleted file mode 100644
index aae07657ea0..00000000000
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ /dev/null
@@ -1,92 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class ProjectTreeRestorer
- LARGE_PROJECT_FILE_SIZE_BYTES = 500.megabyte
-
- attr_reader :user
- attr_reader :shared
- attr_reader :project
-
- def initialize(user:, shared:, project:)
- @user = user
- @shared = shared
- @project = project
- @tree_loader = ProjectTreeLoader.new
- end
-
- def restore
- @tree_hash = read_tree_hash
- @project_members = @tree_hash.delete('project_members')
-
- RelationRenameService.rename(@tree_hash)
-
- if relation_tree_restorer.restore
- import_failure_service.with_retry(action: 'set_latest_merge_request_diff_ids!') do
- @project.merge_requests.set_latest_merge_request_diff_ids!
- end
-
- true
- else
- false
- end
- rescue => e
- @shared.error(e)
- false
- end
-
- private
-
- def large_project?(path)
- File.size(path) >= LARGE_PROJECT_FILE_SIZE_BYTES
- end
-
- def read_tree_hash
- path = File.join(@shared.export_path, 'project.json')
- dedup_entries = large_project?(path) &&
- Feature.enabled?(:dedup_project_import_metadata, project.group)
-
- @tree_loader.load(path, dedup_entries: dedup_entries)
- rescue => e
- Rails.logger.error("Import/Export error: #{e.message}") # rubocop:disable Gitlab/RailsLogger
- raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
- end
-
- def relation_tree_restorer
- @relation_tree_restorer ||= RelationTreeRestorer.new(
- user: @user,
- shared: @shared,
- importable: @project,
- tree_hash: @tree_hash,
- object_builder: object_builder,
- members_mapper: members_mapper,
- relation_factory: relation_factory,
- reader: reader
- )
- end
-
- def members_mapper
- @members_mapper ||= Gitlab::ImportExport::MembersMapper.new(exported_members: @project_members,
- user: @user,
- importable: @project)
- end
-
- def object_builder
- Gitlab::ImportExport::GroupProjectObjectBuilder
- end
-
- def relation_factory
- Gitlab::ImportExport::ProjectRelationFactory
- end
-
- def reader
- @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
- end
-
- def import_failure_service
- @import_failure_service ||= ImportFailureService.new(@project)
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/project_tree_saver.rb b/lib/gitlab/import_export/project_tree_saver.rb
deleted file mode 100644
index 386a4cfdfc6..00000000000
--- a/lib/gitlab/import_export/project_tree_saver.rb
+++ /dev/null
@@ -1,68 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module ImportExport
- class ProjectTreeSaver
- attr_reader :full_path
-
- def initialize(project:, current_user:, shared:, params: {})
- @params = params
- @project = project
- @current_user = current_user
- @shared = shared
- @full_path = File.join(@shared.export_path, ImportExport.project_filename)
- end
-
- def save
- project_tree = tree_saver.serialize(@project, reader.project_tree)
- fix_project_tree(project_tree)
- tree_saver.save(project_tree, @shared.export_path, ImportExport.project_filename)
-
- true
- rescue => e
- @shared.error(e)
- false
- end
-
- private
-
- # Aware that the resulting hash needs to be pure-hash and
- # does not include any AR objects anymore, only objects that run `.to_json`
- def fix_project_tree(project_tree)
- if @params[:description].present?
- project_tree['description'] = @params[:description]
- end
-
- project_tree['project_members'] += group_members_array
-
- RelationRenameService.add_new_associations(project_tree)
- end
-
- def reader
- @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared)
- end
-
- def group_members_array
- group_members.as_json(reader.group_members_tree).each do |group_member|
- group_member['source_type'] = 'Project' # Make group members project members of the future import
- end
- end
-
- def group_members
- return [] unless @current_user.can?(:admin_group, @project.group)
-
- # We need `.where.not(user_id: nil)` here otherwise when a group has an
- # invitee, it would make the following query return 0 rows since a NULL
- # user_id would be present in the subquery
- # See http://stackoverflow.com/questions/129077/not-in-clause-and-null-values
- non_null_user_ids = @project.project_members.where.not(user_id: nil).select(:user_id)
-
- GroupMembersFinder.new(@project.group).execute.where.not(user_id: non_null_user_ids)
- end
-
- def tree_saver
- @tree_saver ||= RelationTreeSaver.new
- end
- end
- end
-end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 9b84ade1525..1797bbad51a 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -69,7 +69,7 @@ module Gitlab
def process_relation_item!(relation_key, relation_definition, relation_index, data_hash)
relation_object = build_relation(relation_key, relation_definition, data_hash)
return unless relation_object
- return if importable_class == Project && group_model?(relation_object)
+ return if importable_class == ::Project && group_model?(relation_object)
relation_object.assign_attributes(importable_class_sym => @importable)
@@ -110,7 +110,7 @@ module Gitlab
excluded_keys: excluded_keys_for_relation(importable_class_sym))
@importable.assign_attributes(params)
- @importable.drop_visibility_level! if importable_class == Project
+ @importable.drop_visibility_level! if importable_class == ::Project
Gitlab::Timeless.timeless(@importable) do
@importable.save!