diff options
Diffstat (limited to 'lib/gitlab/import_export')
11 files changed, 778 insertions, 339 deletions
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index 2d8e25a9f70..f6f65f85599 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -44,7 +44,7 @@ module Gitlab # We want to use AttributesCleaner for these relations instead, in the future this should be removed to make sure # we are using AttributesPermitter for every imported relation. - DISABLED_RELATION_NAMES = %i[user author issuable_sla].freeze + DISABLED_RELATION_NAMES = %i[author issuable_sla].freeze def initialize(config: ImportExport::Config.new.to_h) @config = config diff --git a/lib/gitlab/import_export/base/object_builder.rb b/lib/gitlab/import_export/base/object_builder.rb index 48836729ff6..5e9c8292c1e 100644 --- a/lib/gitlab/import_export/base/object_builder.rb +++ b/lib/gitlab/import_export/base/object_builder.rb @@ -47,15 +47,15 @@ module Gitlab attributes end - private + def find_with_cache(key = cache_key) + return yield unless lru_cache && key - attr_reader :klass, :attributes, :lru_cache, :cache_key + lru_cache[key] ||= yield + end - def find_with_cache - return yield unless lru_cache && cache_key + private - lru_cache[cache_key] ||= yield - end + attr_reader :klass, :attributes, :lru_cache, :cache_key def cache_from_request_store Gitlab::SafeRequestStore[:lru_cache] ||= LruRedux::Cache.new(LRU_CACHE_SIZE) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index febfe00af0b..61b37256964 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize DEFAULT_MAX_BYTES = 10.gigabytes.freeze - TIMEOUT_LIMIT = 60.seconds + TIMEOUT_LIMIT = 210.seconds def initialize(archive_path:, max_bytes: self.class.max_bytes) @archive_path = archive_path diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb new file mode 100644 index 00000000000..f3c392b8c20 --- /dev/null +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -0,0 +1,274 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Group + class RelationTreeRestorer + def initialize( # rubocop:disable Metrics/ParameterLists + user:, + shared:, + relation_reader:, + members_mapper:, + object_builder:, + relation_factory:, + reader:, + importable:, + importable_attributes:, + importable_path: + ) + @user = user + @shared = shared + @importable = importable + @relation_reader = relation_reader + @members_mapper = members_mapper + @object_builder = object_builder + @relation_factory = relation_factory + @reader = reader + @importable_attributes = importable_attributes + @importable_path = importable_path + end + + def restore + ActiveRecord::Base.uncached do + ActiveRecord::Base.no_touching do + update_params! + + BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do + fix_ci_pipelines_not_sorted_on_legacy_project_json! + create_relations! + end + end + end + + # ensure that we have latest version of the restore + @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload + + true + rescue StandardError => e + @shared.error(e) + false + end + + private + + def bulk_insert_enabled + false + end + + # Loops through the tree of models defined in import_export.yml and + # finds them in the imported JSON so they can be instantiated and saved + # in the DB. The structure and relationships between models are guessed from + # the configuration yaml file too. + # Finally, it updates each attribute in the newly imported project/group. + def create_relations! + relations.each do |relation_key, relation_definition| + process_relation!(relation_key, relation_definition) + end + end + + def process_relation!(relation_key, relation_definition) + @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index| + process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + end + end + + def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) + relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash) + return unless relation_object + return if relation_invalid_for_importable?(relation_object) + + relation_object.assign_attributes(importable_class_sym => @importable) + + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do + relation_object.save! + log_relation_creation(@importable, relation_key, relation_object) + end + rescue StandardError => e + import_failure_service.log_import_failure( + source: 'process_relation_item!', + relation_key: relation_key, + relation_index: relation_index, + exception: e) + end + + def import_failure_service + @import_failure_service ||= ImportFailureService.new(@importable) + end + + def relations + @relations ||= + @reader + .attributes_finder + .find_relations_tree(importable_class_sym) + .deep_stringify_keys + end + + def update_params! + params = @importable_attributes.except(*relations.keys.map(&:to_s)) + params = params.merge(present_override_params) + + # Cleaning all imported and overridden params + params = Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: params, + relation_class: importable_class, + excluded_keys: excluded_keys_for_relation(importable_class_sym)) + + @importable.assign_attributes(params) + + modify_attributes + + Gitlab::Timeless.timeless(@importable) do + @importable.save! + end + end + + def present_override_params + # we filter out the empty strings from the overrides + # keeping the default values configured + override_params&.transform_values do |value| + value.is_a?(String) ? value.presence : value + end&.compact + end + + def override_params + @importable_override_params ||= importable_override_params + end + + def importable_override_params + if @importable.respond_to?(:import_data) + @importable.import_data&.data&.fetch('override_params', nil) || {} + else + {} + end + end + + def modify_attributes + # no-op to be overridden on inheritance + end + + def build_relations(relation_key, relation_definition, relation_index, data_hashes) + data_hashes + .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) } + .tap { |entries| entries.compact! } + end + + def build_relation(relation_key, relation_definition, relation_index, data_hash) + # TODO: This is hack to not create relation for the author + # Rather make `RelationFactory#set_note_author` to take care of that + return data_hash if relation_key == 'author' || already_restored?(data_hash) + + # create relation objects recursively for all sub-objects + relation_definition.each do |sub_relation_key, sub_relation_definition| + transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) + end + + relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash)) + + if relation && !relation.valid? + @shared.logger.warn( + message: "[Project/Group Import] Invalid object relation built", + relation_key: relation_key, + relation_index: relation_index, + relation_class: relation.class.name, + error_messages: relation.errors.full_messages.join(". ") + ) + end + + relation + end + + # Since we update the data hash in place as we restore relation items, + # and since we also de-duplicate items, we might encounter items that + # have already been restored in a previous iteration. + def already_restored?(relation_item) + !relation_item.is_a?(Hash) + end + + def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) + sub_data_hash = data_hash[sub_relation_key] + return unless sub_data_hash + + # if object is a hash we can create simple object + # as it means that this is 1-to-1 vs 1-to-many + current_item = + if sub_data_hash.is_a?(Array) + build_relations( + sub_relation_key, + sub_relation_definition, + relation_index, + sub_data_hash).presence + else + build_relation( + sub_relation_key, + sub_relation_definition, + relation_index, + sub_data_hash) + end + + if current_item + data_hash[sub_relation_key] = current_item + else + data_hash.delete(sub_relation_key) + end + end + + def relation_invalid_for_importable?(_relation_object) + false + end + + def excluded_keys_for_relation(relation) + @reader.attributes_finder.find_excluded_keys(relation) + end + + def importable_class + @importable.class + end + + def importable_class_sym + importable_class.to_s.downcase.to_sym + end + + def relation_factory_params(relation_key, relation_index, data_hash) + { + relation_index: relation_index, + relation_sym: relation_key.to_sym, + relation_hash: data_hash, + importable: @importable, + members_mapper: @members_mapper, + object_builder: @object_builder, + user: @user, + excluded_keys: excluded_keys_for_relation(relation_key) + } + end + + # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json + # This should be removed once legacy JSON format is deprecated. + # Ndjson export file will fix the order during project export. + def fix_ci_pipelines_not_sorted_on_legacy_project_json! + return unless @relation_reader.legacy? + + @relation_reader.sort_ci_pipelines_by_id + end + + # Enable logging of each top-level relation creation when Importing + # into a Group if feature flag is enabled + def log_relation_creation(importable, relation_key, relation_object) + root_ancestor_group = importable.try(:root_ancestor) + + return unless root_ancestor_group + return unless root_ancestor_group.instance_of?(::Group) + return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group) + + @shared.logger.info( + importable_type: importable.class.to_s, + importable_id: importable.id, + relation_key: relation_key, + relation_id: relation_object.id, + author_id: relation_object.try(:author_id), + message: '[Project/Group Import] Created new object relation' + ) + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 618ef9a4f43..d815dd284ba 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -178,17 +178,7 @@ included_attributes: - :project_id - :key - :value - label: - - :title - - :color - - :project_id - - :group_id - - :created_at - - :updated_at - - :template - - :description - - :priority - labels: + label: &label_definition - :title - :color - :project_id @@ -198,23 +188,13 @@ included_attributes: - :template - :description - :priority + labels: *label_definition priorities: - :project_id - :priority - :created_at - :updated_at - milestone: - - :iid - - :title - - :project_id - - :group_id - - :description - - :due_date - - :created_at - - :updated_at - - :start_date - - :state - milestones: + milestone: &milestone_definition - :iid - :title - :project_id @@ -225,6 +205,7 @@ included_attributes: - :updated_at - :start_date - :state + milestones: *milestone_definition protected_branches: - :project_id - :name @@ -272,6 +253,385 @@ included_attributes: - :updated_at - :filepath - :link_type + container_expiration_policy: + - :created_at + - :updated_at + - :next_run_at + - :project_id + - :name_regex + - :cadence + - :older_than + - :keep_n + - :enabled + - :name_regex_keep + project_feature: + - :project_id + - :merge_requests_access_level + - :issues_access_level + - :wiki_access_level + - :snippets_access_level + - :builds_access_level + - :created_at + - :updated_at + - :repository_access_level + - :pages_access_level + - :forking_access_level + - :metrics_dashboard_access_level + - :operations_access_level + - :analytics_access_level + - :security_and_compliance_access_level + - :container_registry_access_level + prometheus_metrics: + - :created_at + - :updated_at + - :project_id + - :y_label + - :unit + - :legend + - :title + - :query + - :group + - :dashboard_path + service_desk_setting: + - :project_id + - :issue_template_key + - :project_key + snippets: + - :title + - :content + - :author_id + - :project_id + - :created_at + - :updated_at + - :file_name + - :visibility_level + - :description + project_members: + - :access_level + - :source_type + - :user_id + - :notification_level + - :created_at + - :updated_at + - :created_by_id + - :invite_email + - :invite_accepted_at + - :requested_at + - :expires_at + - :ldap + - :override + merge_request: &merge_request_definition + - :target_branch + - :source_branch + - :source_project_id + - :author_id + - :assignee_id + - :title + - :created_at + - :updated_at + - :state + - :merge_status + - :target_project_id + - :iid + - :description + - :updated_by_id + - :merge_error + - :merge_params + - :merge_when_pipeline_succeeds + - :merge_user_id + - :merge_commit_sha + - :squash_commit_sha + - :in_progress_merge_commit_sha + - :lock_version + - :approvals_before_merge + - :rebase_commit_sha + - :time_estimate + - :squash + - :last_edited_at + - :last_edited_by_id + - :discussion_locked + - :allow_maintainer_to_push + - :merge_ref_sha + - :draft + - :diff_head_sha + - :source_branch_sha + - :target_branch_sha + merge_requests: *merge_request_definition + award_emoji: + - :user_id + - :name + - :awardable_type + - :created_at + - :updated_at + commit_author: + - :name + - :email + committer: + - :name + - :email + events: + - :target_type + - :action + - :author_id + - :fingerprint + - :created_at + - :updated_at + label_links: + - :target_type + - :created_at + - :updated_at + merge_request_diff: + - :state + - :created_at + - :updated_at + - :base_commit_sha + - :real_size + - :head_commit_sha + - :start_commit_sha + - :commits_count + - :files_count + - :sorted + - :diff_type + merge_request_diff_commits: + - :author_name + - :author_email + - :committer_name + - :committer_email + - :relative_order + - :sha + - :authored_date + - :committed_date + - :message + - :trailers + merge_request_diff_files: + - :relative_order + - :new_file + - :renamed_file + - :deleted_file + - :new_path + - :old_path + - :a_mode + - :b_mode + - :too_large + - :binary + - :diff + metrics: + - :created_at + - :updated_at + - :latest_closed_by_id + - :latest_closed_at + - :merged_by_id + - :merged_at + - :latest_build_started_at + - :latest_build_finished_at + - :first_deployed_to_production_at + - :first_comment_at + - :first_commit_at + - :last_commit_at + - :diff_size + - :modified_paths_size + - :commits_count + - :first_approved_at + - :first_reassigned_at + - :added_lines + - :target_project_id + - :removed_lines + notes: + - :note + - :noteable_type + - :author_id + - :created_at + - :updated_at + - :project_id + - :attachment + - :line_code + - :commit_id + - :system + - :st_diff + - :updated_by_id + - :type + - :position + - :original_position + - :change_position + - :resolved_at + - :resolved_by_id + - :resolved_by_push + - :discussion_id + - :confidential + - :last_edited_at + push_event_payload: + - :commit_count + - :action + - :ref_type + - :commit_from + - :commit_to + - :ref + - :commit_title + - :ref_count + resource_label_events: + - :action + - :user_id + - :created_at + suggestions: + - :relative_order + - :applied + - :commit_id + - :from_content + - :to_content + - :outdated + - :lines_above + - :lines_below + system_note_metadata: + - :commit_count + - :action + - :created_at + - :updated_at + timelogs: + - :time_spent + - :user_id + - :project_id + - :spent_at + - :created_at + - :updated_at + - :summary + external_pull_request: &external_pull_request_definition + - :created_at + - :updated_at + - :project_id + - :pull_request_iid + - :status + - :source_branch + - :target_branch + - :source_repository + - :target_repository + - :source_sha + - :target_sha + external_pull_requests: *external_pull_request_definition + statuses: + - :project_id + - :status + - :finished_at + - :created_at + - :updated_at + - :started_at + - :coverage + - :commit_id + - :name + - :options + - :allow_failure + - :stage + - :stage_idx + - :tag + - :ref + - :user_id + - :type + - :target_url + - :description + - :erased_at + - :artifacts_expire_at + - :environment + - :yaml_variables + - :queued_at + - :lock_version + - :coverage_regex + - :retried + - :protected + - :failure_reason + - :scheduled_at + - :scheduling_type + ci_pipelines: + - :ref + - :sha + - :before_sha + - :created_at + - :updated_at + - :tag + - :yaml_errors + - :committed_at + - :project_id + - :status + - :started_at + - :finished_at + - :duration + - :user_id + - :lock_version + - :source + - :protected + - :config_source + - :failure_reason + - :iid + - :source_sha + - :target_sha + stages: + - :name + - :status + - :position + - :lock_version + - :project_id + - :created_at + - :updated_at + actions: + - :event + - :image_v432x230 + design: &design_definition + - :iid + - :project_id + - :filename + - :relative_position + designs: *design_definition + design_versions: + - :created_at + - :sha + - :author_id + issue_assignees: + - :user_id + sentry_issue: + - :sentry_issue_identifier + zoom_meetings: + - :project_id + - :issue_status + - :url + - :created_at + - :updated_at + issues: + - :title + - :author_id + - :project_id + - :created_at + - :updated_at + - :description + - :state + - :iid + - :updated_by_id + - :confidential + - :closed_at + - :closed_by_id + - :due_date + - :lock_version + - :weight + - :time_estimate + - :relative_position + - :external_author + - :last_edited_at + - :last_edited_by_id + - :discussion_locked + - :health_status + - :external_key + - :issue_type + group_members: + - :access_level + - :source_type + - :user_id + - :notification_level + - :created_at + - :updated_at + - :created_by_id + - :invite_email + - :invite_accepted_at + - :requested_at + - :expires_at + - :ldap + - :override # Do not include the following attributes for the models specified. excluded_attributes: @@ -387,16 +747,7 @@ excluded_attributes: - :service_desk_reply_to - :upvotes_count - :work_item_type_id - merge_request: - - :milestone_id - - :sprint_id - - :ref_fetched - - :merge_jid - - :rebase_jid - - :latest_merge_request_diff_id - - :head_pipeline_id - - :state_id - merge_requests: + merge_request: &merge_request_excluded_definition - :milestone_id - :sprint_id - :ref_fetched @@ -405,6 +756,7 @@ excluded_attributes: - :latest_merge_request_diff_id - :head_pipeline_id - :state_id + merge_requests: *merge_request_excluded_definition award_emoji: - :awardable_id statuses: @@ -473,10 +825,9 @@ excluded_attributes: - :issue_id zoom_meetings: - :issue_id - design: - - :issue_id - designs: + design: &design_excluded_definition - :issue_id + designs: *design_excluded_definition design_versions: - :issue_id actions: @@ -660,4 +1011,13 @@ ee: - :name - :created_at - :updated_at - + project_feature: + - :requirements_access_level + security_setting: + - :project_id + - :created_at + - :updated_at + - :auto_fix_container_scanning + - :auto_fix_dast + - :auto_fix_dependency_scanning + - :auto_fix_sast diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index b03dceba303..f7598ba1337 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -29,6 +29,7 @@ module Gitlab def find return if epic? && group.nil? return find_diff_commit_user if diff_commit_user? + return find_diff_commit if diff_commit? super end @@ -83,9 +84,38 @@ module Gitlab end def find_diff_commit_user - find_with_cache do - MergeRequest::DiffCommitUser - .find_or_create(@attributes['name'], @attributes['email']) + find_or_create_diff_commit_user(@attributes['name'], @attributes['email']) + end + + def find_diff_commit + row = @attributes.dup + + # Diff commits come in two formats: + # + # 1. The old format where author/committer details are separate fields + # 2. The new format where author/committer details are nested objects, + # and pre-processed by `find_diff_commit_user`. + # + # The code here ensures we support both the old and new format. + aname = row.delete('author_name') + amail = row.delete('author_email') + cname = row.delete('committer_name') + cmail = row.delete('committer_email') + author = row.delete('commit_author') + committer = row.delete('committer') + + row['commit_author'] = author || + find_or_create_diff_commit_user(aname, amail) + + row['committer'] = committer || + find_or_create_diff_commit_user(cname, cmail) + + MergeRequestDiffCommit.new(row) + end + + def find_or_create_diff_commit_user(name, email) + find_with_cache([MergeRequest::DiffCommitUser, name, email]) do + MergeRequest::DiffCommitUser.find_or_create(name, email) end end @@ -113,6 +143,10 @@ module Gitlab klass == MergeRequest::DiffCommitUser end + def diff_commit? + klass == MergeRequestDiffCommit + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 888a5a10f2c..d84db92fe69 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -33,7 +33,8 @@ module Gitlab links: 'Releases::Link', metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', - committer: 'MergeRequest::DiffCommitUser' }.freeze + committer: 'MergeRequest::DiffCommitUser', + merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze BUILD_MODELS = %i[Ci::Build commit_status].freeze @@ -59,6 +60,7 @@ module Gitlab external_pull_requests DesignManagement::Design MergeRequest::DiffCommitUser + MergeRequestDiffCommit ].freeze def create diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb new file mode 100644 index 00000000000..6e9548f393a --- /dev/null +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer + # Relations which cannot be saved at project level (and have a group assigned) + GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze + + private + + def bulk_insert_enabled + true + end + + def modify_attributes + @importable.reconcile_shared_runners_setting! + @importable.drop_visibility_level! + end + + def relation_invalid_for_importable?(relation_object) + GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb index 4db92b12968..034122a9f14 100644 --- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -4,7 +4,7 @@ module Gitlab module ImportExport module Project module Sample - class RelationTreeRestorer < ImportExport::RelationTreeRestorer + class RelationTreeRestorer < ImportExport::Project::RelationTreeRestorer def initialize(...) super(...) @@ -18,10 +18,10 @@ module Gitlab end def dates - return [] if relation_reader.legacy? + return [] if @relation_reader.legacy? RelationFactory::DATE_MODELS.flat_map do |tag| - relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| + @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| model.first['due_date'] end end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1f0fa249390..aafed850afa 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -6,20 +6,16 @@ module Gitlab class TreeSaver attr_reader :full_path - def initialize(project:, current_user:, shared:, params: {}) + def initialize(project:, current_user:, shared:, params: {}, logger: Gitlab::Import::Logger) @params = params @project = project @current_user = current_user @shared = shared + @logger = logger end def save - ImportExport::Json::StreamingSerializer.new( - exportable, - reader.project_tree, - json_writer, - exportable_path: "project" - ).execute + stream_export true rescue StandardError => e @@ -31,6 +27,32 @@ module Gitlab private + def stream_export + on_retry = proc do |exception, try, elapsed_time, next_interval| + @logger.info( + message: "Project export retry triggered from streaming", + 'error.class': exception.class, + 'error.message': exception.message, + try_count: try, + elapsed_time_s: elapsed_time, + wait_to_retry_s: next_interval, + project_name: @project.name, + project_id: @project.id + ) + end + + serializer = ImportExport::Json::StreamingSerializer.new( + exportable, + reader.project_tree, + json_writer, + exportable_path: "project" + ) + + Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do + serializer.execute + end + end + def reader @reader ||= Gitlab::ImportExport::Reader.new(shared: @shared) end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb deleted file mode 100644 index 1eeacafef53..00000000000 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ /dev/null @@ -1,280 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - class RelationTreeRestorer - # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze - - attr_reader :user - attr_reader :shared - attr_reader :importable - attr_reader :relation_reader - - def initialize( # rubocop:disable Metrics/ParameterLists - user:, shared:, relation_reader:, - members_mapper:, object_builder:, - relation_factory:, - reader:, - importable:, - importable_attributes:, - importable_path: - ) - @user = user - @shared = shared - @importable = importable - @relation_reader = relation_reader - @members_mapper = members_mapper - @object_builder = object_builder - @relation_factory = relation_factory - @reader = reader - @importable_attributes = importable_attributes - @importable_path = importable_path - end - - def restore - ActiveRecord::Base.uncached do - ActiveRecord::Base.no_touching do - update_params! - - BulkInsertableAssociations.with_bulk_insert(enabled: project?) do - fix_ci_pipelines_not_sorted_on_legacy_project_json! - create_relations! - end - end - end - - # ensure that we have latest version of the restore - @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload - - true - rescue StandardError => e - @shared.error(e) - false - end - - private - - def project? - @importable.instance_of?(::Project) - end - - # Loops through the tree of models defined in import_export.yml and - # finds them in the imported JSON so they can be instantiated and saved - # in the DB. The structure and relationships between models are guessed from - # the configuration yaml file too. - # Finally, it updates each attribute in the newly imported project/group. - def create_relations! - relations.each do |relation_key, relation_definition| - process_relation!(relation_key, relation_definition) - end - end - - def process_relation!(relation_key, relation_definition) - @relation_reader.consume_relation(@importable_path, relation_key).each do |data_hash, relation_index| - process_relation_item!(relation_key, relation_definition, relation_index, data_hash) - end - end - - def process_relation_item!(relation_key, relation_definition, relation_index, data_hash) - relation_object = build_relation(relation_key, relation_definition, relation_index, data_hash) - return unless relation_object - return if project? && group_model?(relation_object) - - relation_object.assign_attributes(importable_class_sym => @importable) - - import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do - relation_object.save! - log_relation_creation(@importable, relation_key, relation_object) - end - rescue StandardError => e - import_failure_service.log_import_failure( - source: 'process_relation_item!', - relation_key: relation_key, - relation_index: relation_index, - exception: e) - end - - def import_failure_service - @import_failure_service ||= ImportFailureService.new(@importable) - end - - def relations - @relations ||= - @reader - .attributes_finder - .find_relations_tree(importable_class_sym) - .deep_stringify_keys - end - - def update_params! - params = @importable_attributes.except(*relations.keys.map(&:to_s)) - params = params.merge(present_override_params) - - # Cleaning all imported and overridden params - params = Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, - relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) - - @importable.assign_attributes(params) - - modify_attributes - - Gitlab::Timeless.timeless(@importable) do - @importable.save! - end - end - - def present_override_params - # we filter out the empty strings from the overrides - # keeping the default values configured - override_params&.transform_values do |value| - value.is_a?(String) ? value.presence : value - end&.compact - end - - def override_params - @importable_override_params ||= importable_override_params - end - - def importable_override_params - if @importable.respond_to?(:import_data) - @importable.import_data&.data&.fetch('override_params', nil) || {} - else - {} - end - end - - def modify_attributes - return unless project? - - @importable.reconcile_shared_runners_setting! - @importable.drop_visibility_level! - end - - def build_relations(relation_key, relation_definition, relation_index, data_hashes) - data_hashes - .map { |data_hash| build_relation(relation_key, relation_definition, relation_index, data_hash) } - .tap { |entries| entries.compact! } - end - - def build_relation(relation_key, relation_definition, relation_index, data_hash) - # TODO: This is hack to not create relation for the author - # Rather make `RelationFactory#set_note_author` to take care of that - return data_hash if relation_key == 'author' || already_restored?(data_hash) - - # create relation objects recursively for all sub-objects - relation_definition.each do |sub_relation_key, sub_relation_definition| - transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) - end - - relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash)) - - if relation && !relation.valid? - @shared.logger.warn( - message: "[Project/Group Import] Invalid object relation built", - relation_key: relation_key, - relation_index: relation_index, - relation_class: relation.class.name, - error_messages: relation.errors.full_messages.join(". ") - ) - end - - relation - end - - # Since we update the data hash in place as we restore relation items, - # and since we also de-duplicate items, we might encounter items that - # have already been restored in a previous iteration. - def already_restored?(relation_item) - !relation_item.is_a?(Hash) - end - - def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) - sub_data_hash = data_hash[sub_relation_key] - return unless sub_data_hash - - # if object is a hash we can create simple object - # as it means that this is 1-to-1 vs 1-to-many - current_item = - if sub_data_hash.is_a?(Array) - build_relations( - sub_relation_key, - sub_relation_definition, - relation_index, - sub_data_hash).presence - else - build_relation( - sub_relation_key, - sub_relation_definition, - relation_index, - sub_data_hash) - end - - if current_item - data_hash[sub_relation_key] = current_item - else - data_hash.delete(sub_relation_key) - end - end - - def group_model?(relation_object) - GROUP_MODELS.include?(relation_object.class) && relation_object.group_id - end - - def excluded_keys_for_relation(relation) - @reader.attributes_finder.find_excluded_keys(relation) - end - - def importable_class - @importable.class - end - - def importable_class_sym - importable_class.to_s.downcase.to_sym - end - - def relation_factory_params(relation_key, relation_index, data_hash) - { - relation_index: relation_index, - relation_sym: relation_key.to_sym, - relation_hash: data_hash, - importable: @importable, - members_mapper: @members_mapper, - object_builder: @object_builder, - user: @user, - excluded_keys: excluded_keys_for_relation(relation_key) - } - end - - # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json - # This should be removed once legacy JSON format is deprecated. - # Ndjson export file will fix the order during project export. - def fix_ci_pipelines_not_sorted_on_legacy_project_json! - return unless relation_reader.legacy? - - relation_reader.sort_ci_pipelines_by_id - end - - # Enable logging of each top-level relation creation when Importing - # into a Group if feature flag is enabled - def log_relation_creation(importable, relation_key, relation_object) - root_ancestor_group = importable.try(:root_ancestor) - - return unless root_ancestor_group - return unless root_ancestor_group.instance_of?(::Group) - return unless Feature.enabled?(:log_import_export_relation_creation, root_ancestor_group) - - @shared.logger.info( - importable_type: importable.class.to_s, - importable_id: importable.id, - relation_key: relation_key, - relation_id: relation_object.id, - author_id: relation_object.try(:author_id), - message: '[Project/Group Import] Created new object relation' - ) - end - end - end -end |