diff options
Diffstat (limited to 'lib/gitlab/import_export')
19 files changed, 217 insertions, 336 deletions
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 8843b4f5755..dea989931c7 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders, + :import_only_tree def initialize(config:) @tree = config[:tree] || {} @@ -13,13 +14,16 @@ module Gitlab @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} @include_if_exportable = config[:include_if_exportable] || {} + @import_only_tree = config[:import_only_tree] || {} end def find_root(model_key) find(model_key, @tree[model_key]) end - def find_relations_tree(model_key) + def find_relations_tree(model_key, include_import_only_tree: false) + return @tree[model_key].deep_merge(@import_only_tree[model_key] || {}) if include_import_only_tree + @tree[model_key] end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index 8c7a6c13246..889cab88de4 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -80,7 +80,7 @@ module Gitlab # Deep traverse relations tree to build a list of allowed model relations def build_associations - stack = @attributes_finder.tree.to_a + stack = @attributes_finder.tree.deep_merge(@attributes_finder.import_only_tree).to_a while stack.any? model_name, relations = stack.pop diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index e3813070aa4..3d96e891797 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -295,6 +295,13 @@ module Gitlab end def unique_relation? + # this guard is necessary because + # when multiple approval_project_rules_protected_branch referenced the same protected branch + # or approval_project_rules_user referenced the same user + # the different instances were squashed into one + # because this method returned true for reason that needs investigation + return if @relation_sym == :approval_rules + strong_memoize(:unique_relation) do importable_foreign_key.present? && (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 77b85fc9f15..986191bdb6b 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -17,6 +17,8 @@ module Gitlab BATCH_SIZE = 100 MIN_RECORDS_SIZE = 1 + attr_reader :invalid_subrelations + # @param relation_object [Object] Object of a project/group, e.g. an issue # @param relation_key [String] Name of the object association to group/project, e.g. :issues # @param relation_definition [Hash] Object subrelations as defined in import_export.yml @@ -43,14 +45,11 @@ module Gitlab relation_object.save! save_subrelations - ensure - log_invalid_subrelations end private - attr_reader :relation_object, :relation_key, :relation_definition, - :importable, :collection_subrelations, :invalid_subrelations + attr_reader :relation_object, :relation_key, :relation_definition, :importable, :collection_subrelations # rubocop:disable GitlabSecurity/PublicSend def save_subrelations @@ -92,30 +91,6 @@ module Gitlab end end # rubocop:enable GitlabSecurity/PublicSend - - def log_invalid_subrelations - invalid_subrelations.flatten.each do |record| - Gitlab::Import::Logger.info( - message: '[Project/Group Import] Invalid subrelation', - importable_column_name => importable.id, - relation_key: relation_key, - error_messages: record.errors.full_messages.to_sentence - ) - - ImportFailure.create( - source: 'RelationObjectSaver#save!', - relation_key: relation_key, - exception_class: 'RecordInvalid', - exception_message: record.errors.full_messages.to_sentence, - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, - importable_column_name => importable.id - ) - end - end - - def importable_column_name - @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym - end end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 64ef3dd4830..d681f39f00b 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -90,6 +90,7 @@ module Gitlab def untar_with_options(archive:, dir:, options:) execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) + remove_symlinks(dir) end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -120,6 +121,19 @@ module Gitlab FileUtils.copy_entry(source, destination) true end + + def remove_symlinks(dir) + ignore_file_names = %w[. ..] + + # Using File::FNM_DOTMATCH to also delete symlinks starting with "." + Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + .reject { |f| ignore_file_names.include?(File.basename(f)) } + .each do |filepath| + FileUtils.rm(filepath) if File.lstat(filepath).symlink? + end + + true + end end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 83c4bc47349..e1a62e3b25a 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -10,6 +10,7 @@ module Gitlab @ee_hash = @hash.delete(:ee) || {} @hash[:tree] = normalize_tree(@hash[:tree]) + @hash[:import_only_tree] = normalize_tree(@hash[:import_only_tree] || {}) @ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {}) end @@ -51,7 +52,7 @@ module Gitlab end def parse_yaml - YAML.load_file(@config) + YAML.safe_load_file(@config, aliases: true, permitted_classes: [Symbol]) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 1878b5b1a30..d2593289c23 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -8,7 +8,6 @@ module Gitlab ImporterError = Class.new(StandardError) MAX_RETRIES = 8 - IGNORED_FILENAMES = %w(. ..).freeze def self.import(*args, **kwargs) new(*args, **kwargs).import @@ -24,7 +23,7 @@ module Gitlab mkdir_p(@shared.export_path) mkdir_p(@shared.archive_path) - remove_symlinks + remove_symlinks(@shared.export_path) copy_archive wait_for_archived_file do @@ -36,7 +35,7 @@ module Gitlab false ensure remove_import_file - remove_symlinks + remove_symlinks(@shared.export_path) end private @@ -86,22 +85,10 @@ module Gitlab end end - def remove_symlinks - extracted_files.each do |path| - FileUtils.rm(path) if File.lstat(path).symlink? - end - - true - end - def remove_import_file FileUtils.rm_rf(@archive_file) end - def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } - end - def validate_decompressed_archive_size raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index 5a78f2fb531..5453792e904 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -34,7 +34,6 @@ module Gitlab update_params! BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do - fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end end @@ -90,13 +89,23 @@ module Gitlab def save_relation_object(relation_object, relation_key, relation_definition, relation_index) if relation_object.new_record? - Gitlab::ImportExport::Base::RelationObjectSaver.new( + saver = Gitlab::ImportExport::Base::RelationObjectSaver.new( relation_object: relation_object, relation_key: relation_key, relation_definition: relation_definition, importable: @importable - ).execute + ) + + saver.execute + + log_invalid_subrelations(saver.invalid_subrelations, relation_key) else + if relation_object.invalid? + Gitlab::Import::Errors.merge_nested_errors(relation_object) + + raise(ActiveRecord::RecordInvalid, relation_object) + end + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end @@ -113,7 +122,7 @@ module Gitlab @relations ||= @reader .attributes_finder - .find_relations_tree(importable_class_sym) + .find_relations_tree(importable_class_sym, include_import_only_tree: true) .deep_stringify_keys end @@ -126,9 +135,7 @@ module Gitlab modify_attributes - Gitlab::Timeless.timeless(@importable) do - @importable.save! - end + @importable.save!(touch: false) end def filter_attributes(params) @@ -265,15 +272,6 @@ module Gitlab } end - # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json - # This should be removed once legacy JSON format is deprecated. - # Ndjson export file will fix the order during project export. - def fix_ci_pipelines_not_sorted_on_legacy_project_json! - return unless @relation_reader.legacy? - - @relation_reader.sort_ci_pipelines_by_id - end - # Enable logging of each top-level relation creation when Importing into a Group def log_relation_creation(importable, relation_key, relation_object) root_ancestor_group = importable.try(:root_ancestor) @@ -290,6 +288,32 @@ module Gitlab message: '[Project/Group Import] Created new object relation' ) end + + def log_invalid_subrelations(invalid_subrelations, relation_key) + invalid_subrelations.flatten.each do |record| + Gitlab::Import::Errors.merge_nested_errors(record) + + @shared.logger.info( + message: '[Project/Group Import] Invalid subrelation', + importable_column_name => @importable.id, + relation_key: relation_key, + error_messages: record.errors.full_messages.to_sentence + ) + + ::ImportFailure.create( + source: 'RelationTreeRestorer#save_relation_object', + relation_key: relation_key, + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: record.errors.full_messages.to_sentence, + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + importable_column_name => @importable.id + ) + end + end + + def importable_column_name + @column_name ||= @importable.class.reflect_on_association(:import_failures).foreign_key.to_sym + end end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb deleted file mode 100644 index ee360020556..00000000000 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ /dev/null @@ -1,123 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Json - class LegacyReader - class File < LegacyReader - include Gitlab::Utils::StrongMemoize - - def initialize(path, relation_names:, allowed_path: nil) - @path = path - super( - relation_names: relation_names, - allowed_path: allowed_path) - end - - def exist? - ::File.exist?(@path) - end - - protected - - def tree_hash - strong_memoize(:tree_hash) do - read_hash - end - end - - def read_hash - Gitlab::Json.parse(::File.read(@path)) - rescue StandardError => e - Gitlab::ErrorTracking.log_exception(e) - raise Gitlab::ImportExport::Error, 'Incorrect JSON format' - end - end - - class Hash < LegacyReader - def initialize(tree_hash, relation_names:, allowed_path: nil) - @tree_hash = tree_hash - super( - relation_names: relation_names, - allowed_path: allowed_path) - end - - def exist? - @tree_hash.present? - end - - protected - - attr_reader :tree_hash - end - - def initialize(relation_names:, allowed_path:) - @relation_names = relation_names.map(&:to_s) - @consumed_relations = Set.new - - # This is legacy reader, to be used in transition - # period before `.ndjson`, - # we strong validate what is being readed - @allowed_path = allowed_path - end - - def exist? - raise NotImplementedError - end - - def legacy? - true - end - - def consume_attributes(importable_path) - unless importable_path == @allowed_path - raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead." - end - - attributes - end - - def consume_relation(importable_path, key) - unless importable_path == @allowed_path - raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead." - end - - Enumerator.new do |documents| - next unless @consumed_relations.add?("#{importable_path}/#{key}") - - value = relations.delete(key) - next if value.nil? - - if value.is_a?(Array) - value.each.with_index do |item, idx| - documents << [item, idx] - end - else - documents << [value, 0] - end - end - end - - def sort_ci_pipelines_by_id - relations['ci_pipelines']&.sort_by! { |hash| hash['id'] } - end - - private - - attr_reader :relation_names, :allowed_path - - def tree_hash - raise NotImplementedError - end - - def attributes - @attributes ||= tree_hash.slice!(*relation_names) - end - - def relations - @relations ||= tree_hash.extract!(*relation_names) - end - end - end - end -end diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb deleted file mode 100644 index e03ab9f7650..00000000000 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Json - class LegacyWriter - include Gitlab::ImportExport::CommandLineUtil - - attr_reader :path - - def initialize(path, allowed_path:) - @path = path - @keys = Set.new - - # This is legacy writer, to be used in transition - # period before `.ndjson`, - # we strong validate what is being written - @allowed_path = allowed_path - - mkdir_p(File.dirname(@path)) - file.write('{}') - end - - def close - @file&.close - @file = nil - end - - def write_attributes(exportable_path, hash) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - hash.each do |key, value| - write(key, value) - end - end - - def write_relation(exportable_path, key, value) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - write(key, value) - end - - def write_relation_array(exportable_path, key, items) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - write(key, []) - - # rewind by two bytes, to overwrite ']}' - file.pos = file.size - 2 - - items.each_with_index do |item, idx| - file.write(',') if idx > 0 - file.write(item.to_json) - end - - file.write(']}') - end - - private - - def write(key, value) - raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) - - # rewind by one byte, to overwrite '}' - file.pos = file.size - 1 - - file.write(',') if @keys.any? - file.write(key.to_json) - file.write(':') - file.write(value.to_json) - file.write('}') - - @keys.add(key) - end - - def file - @file ||= File.open(@path, "wb") - end - end - end - end -end diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 510da61d3ab..3de56aacf18 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -17,14 +17,12 @@ module Gitlab Dir.exist?(@dir_path) end - # This can be removed once legacy_reader is deprecated. - def legacy? - false - end - def consume_attributes(importable_path) # This reads from `tree/project.json` path = file_path("#{importable_path}.json") + + raise Gitlab::ImportExport::Error, 'Invalid file' if !File.exist?(path) || File.symlink?(path) + data = File.read(path, MAX_JSON_DOCUMENT_SIZE) json_decode(data) end @@ -36,7 +34,7 @@ module Gitlab # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") - next unless File.exist?(path) + next if !File.exist?(path) || File.symlink?(path) File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num| documents << [json_decode(line), line_num] diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 389ab8b4c97..9bb0770dc90 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -8,6 +8,8 @@ module Gitlab BATCH_SIZE = 100 + attr_reader :exported_objects_count + class Raw < String def to_json(*_args) to_s @@ -21,6 +23,7 @@ module Gitlab @relations_schema = relations_schema @json_writer = json_writer @logger = logger + @exported_objects_count = 0 end def execute @@ -40,21 +43,28 @@ module Gitlab relations_schema.merge(include: nil, preloads: nil, unsafe: true)) json_writer.write_attributes(exportable_path, attributes) + + increment_exported_objects_counter end - def serialize_relation(definition) + def serialize_relation(definition, options = {}) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? - key, options = definition.first + key, definition_options = definition.first record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend + + if options[:batch_ids] + record = record.where(record.model.primary_key => Array.wrap(options[:batch_ids]).map(&:to_i)) + end + if record.is_a?(ActiveRecord::Relation) - serialize_many_relations(key, record, options) + serialize_many_relations(key, record, definition_options) elsif record.respond_to?(:each) # this is to support `project_members` that return an Array - serialize_many_each(key, record, options) + serialize_many_each(key, record, definition_options) else - serialize_single_relation(key, record, options) + serialize_single_relation(key, record, definition_options) end end @@ -76,6 +86,8 @@ module Gitlab items << exportable_json_record(record, options, key) + increment_exported_objects_counter + after_read_callback(record) end end @@ -175,6 +187,8 @@ module Gitlab enumerator = Enumerator.new do |items| records.each do |record| items << exportable_json_record(record, options, key) + + increment_exported_objects_counter end end @@ -187,6 +201,8 @@ module Gitlab json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) + + increment_exported_objects_counter end def includes @@ -263,6 +279,10 @@ module Gitlab message += ". Number of records to export: #{size}" if size logger.info(message: message, **log_base_data) end + + def increment_exported_objects_counter + @exported_objects_count += 1 + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d97ffee8698..36a3c73271b 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -89,13 +89,15 @@ tree: - :milestone - :resource_state_events - :external_pull_requests + - commit_notes: + - :author + - events: + - :push_event_payload - ci_pipelines: - - notes: - - :author - - events: - - :push_event_payload - stages: - - :statuses + - :builds + - :generic_commit_statuses + - :bridges - :external_pull_request - :merge_request - :pipeline_metadata @@ -119,9 +121,25 @@ tree: - label: - :priorities - :service_desk_setting + - :design_management_repository group_members: - :user +# Used to support old exports that were exported before the removal/rename of the associations +# +# For example, statuses of ci_pipelines are no longer exported, and instead, statuses are +# exported as builds, generic_commit_statuses, and bridges. So in order to allow statuses +# to be still imported, it is added to the list below. +import_only_tree: + project: + - ci_pipelines: + - notes: + - :author + - events: + - :push_event_payload + - stages: + - :statuses + # Only include the following attributes for the models specified. included_attributes: user: @@ -529,7 +547,7 @@ included_attributes: - :source_sha - :target_sha external_pull_requests: *external_pull_request_definition - statuses: + statuses: &statuses_definition - :project_id - :status - :finished_at @@ -562,6 +580,9 @@ included_attributes: - :scheduled_at - :scheduling_type - :ci_stage + builds: *statuses_definition + generic_commit_statuses: *statuses_definition + bridges: *statuses_definition ci_pipelines: - :ref - :sha @@ -596,6 +617,7 @@ included_attributes: - :project_id - :created_at - :updated_at + # - :statuses # old exports use statuses instead of builds, generic_commit_statuses and bridges actions: - :event design: &design_definition @@ -896,7 +918,7 @@ excluded_attributes: merge_requests: *merge_request_excluded_definition award_emoji: - :awardable_id - statuses: + statuses: &statuses_excluded_definition - :trace - :token - :token_encrypted @@ -918,6 +940,9 @@ excluded_attributes: - :processed - :id_convert_to_bigint - :stage_id_convert_to_bigint + builds: *statuses_excluded_definition + generic_commit_statuses: *statuses_excluded_definition + bridges: *statuses_excluded_definition sentry_issue: - :issue_id push_event_payload: @@ -955,6 +980,9 @@ excluded_attributes: notes: - :noteable_id - :review_id + commit_notes: + - :noteable_id + - :review_id label_links: - :label_id - :target_id @@ -1079,6 +1107,8 @@ methods: - :squash_option notes: - :type + commit_notes: + - :type labels: - :type label: @@ -1100,8 +1130,6 @@ methods: - :type lists: - :list_type - ci_pipelines: - - :notes issues: - :state @@ -1111,10 +1139,11 @@ methods: preloads: issues: project: :route - statuses: - # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` - # tags: # needed by tag_list - project: # deprecated: needed by coverage_regex of Ci::Build + builds: + metadata: + project: + bridges: + metadata: merge_requests: source_project: :route # needed by source_branch_sha and diff_head_sha target_project: :route # needed by target_branch_sha @@ -1167,6 +1196,9 @@ ee: - :milestone - lists: - :milestone + - approval_rules: + - :approval_project_rules_protected_branches + - :approval_project_rules_users included_attributes: issuable_sla: @@ -1232,9 +1264,30 @@ ee: - :description iterations_cadence: - :title + approval_rules: + - :approvals_required + - :name + - :rule_type + - :scanners + - :vulnerabilities_allowed + - :severity_levels + - :report_type + - :vulnerability_states + - :orchestration_policy_idx + - :applies_to_all_protected_branches + approval_project_rules_protected_branches: + - :protected_branch + approval_project_rules_users: + - :user_id excluded_attributes: project: - :vulnerability_hooks_integrations + approval_rules: + - :created_at + - :updated_at + methods: + approval_project_rules_protected_branches: + - :branch_name preloads: issues: epic: diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 50a67a746f8..ac28ae6bfe0 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -60,10 +60,11 @@ module Gitlab def prepare_attributes attributes.dup.tap do |atts| - atts.delete('group') unless epic? || iteration? + atts.delete('group') unless group_level_object? if label? atts['type'] = 'ProjectLabel' # Always create project labels + atts.delete('group_id') elsif milestone? if atts['group_id'] # Transform new group milestones into project ones atts['iid'] = nil @@ -141,10 +142,6 @@ module Gitlab klass == MergeRequestDiffCommit end - def iteration? - klass == Iteration - end - # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -163,7 +160,11 @@ module Gitlab end def group_relation_without_group? - (epic? || iteration?) && group.nil? + group_level_object? && group.nil? + end + + def group_level_object? + epic? end end end diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 4134c428500..5d7e3ea9ed7 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -5,6 +5,7 @@ module Gitlab module Project class RelationFactory < Base::RelationFactory OVERRIDES = { snippets: :project_snippets, + commit_notes: 'Note', ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', stages: 'Ci::Stage', @@ -12,6 +13,7 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + bridges: 'Ci::Bridge', runners: 'Ci::Runner', pipeline_metadata: 'Ci::PipelineMetadata', hooks: 'ProjectHook', @@ -20,6 +22,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', design: 'DesignManagement::Design', designs: 'DesignManagement::Design', + design_management_repository: 'DesignManagement::Repository', design_versions: 'DesignManagement::Version', actions: 'DesignManagement::Action', labels: :project_labels, @@ -37,7 +40,7 @@ module Gitlab committer: 'MergeRequest::DiffCommitUser', merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze - BUILD_MODELS = %i[Ci::Build commit_status].freeze + BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze GROUP_REFERENCES = %w[group_id].freeze @@ -83,13 +86,14 @@ module Gitlab def setup_models case @relation_name when :merge_request_diff_files then setup_diff - when :notes then setup_note + when :notes, :Note then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build when :issues then setup_issue when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level + when :ApprovalProjectRulesProtectedBranch then setup_merge_approval_protected_branch when :releases then setup_release end @@ -142,9 +146,22 @@ module Gitlab def setup_pipeline @relation_hash.fetch('stages', []).each do |stage| + # old export files have statuses stage.statuses.each do |status| status.pipeline = imported_object end + + stage.builds.each do |status| + status.pipeline = imported_object + end + + stage.bridges.each do |status| + status.pipeline = imported_object + end + + stage.generic_commit_statuses.each do |status| + status.pipeline = imported_object + end end end @@ -180,6 +197,13 @@ module Gitlab root_ancestor.max_member_access_for_user(@user) == Gitlab::Access::OWNER end + def setup_merge_approval_protected_branch + source_branch_name = @relation_hash.delete('branch_name') + target_branch = @importable.protected_branches.find_by(name: source_branch_name) + + @relation_hash['protected_branch'] = target_branch + end + def compute_relative_position return unless max_relative_position diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb index 47196db6f8a..b5247754199 100644 --- a/lib/gitlab/import_export/project/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -5,10 +5,14 @@ module Gitlab module Project class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze private + def group_models + GROUP_MODELS + end + def bulk_insert_enabled true end @@ -19,9 +23,11 @@ module Gitlab end def relation_invalid_for_importable?(relation_object) - GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + group_models.include?(relation_object.class) && relation_object.group_id end end end end end + +Gitlab::ImportExport::Project::RelationTreeRestorer.prepend_mod diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb index 034122a9f14..639f34980ff 100644 --- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -18,8 +18,6 @@ module Gitlab end def dates - return [] if @relation_reader.legacy? - RelationFactory::DATE_MODELS.flat_map do |tag| @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| model.first['due_date'] diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 47f82a901b7..e791424875a 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -17,7 +17,7 @@ module Gitlab end def restore - unless relation_reader + unless relation_reader.exist? raise Gitlab::ImportExport::Error, 'invalid import format' end @@ -47,28 +47,11 @@ module Gitlab private def relation_reader - strong_memoize(:relation_reader) do - [ndjson_relation_reader, legacy_relation_reader] - .compact.find(&:exist?) - end - end - - def ndjson_relation_reader - return unless Feature.enabled?(:project_import_ndjson, project.namespace) - - ImportExport::Json::NdjsonReader.new( + @relation_reader ||= ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end - def legacy_relation_reader - ImportExport::Json::LegacyReader::File.new( - File.join(shared.export_path, 'project.json'), - relation_names: reader.project_relation_names, - allowed_path: importable_path - ) - end - def relation_tree_restorer @relation_tree_restorer ||= relation_tree_restorer_class.new( user: @user, diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 05b96f7e8ce..fd5fa73764e 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -81,13 +81,10 @@ module Gitlab end def json_writer - @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) - full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) - else - full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') - end + @json_writer ||= begin + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) + end end end end |