From 4555e1b21c365ed8303ffb7a3325d773c9b8bf31 Mon Sep 17 00:00:00 2001 From: GitLab Bot Date: Wed, 19 May 2021 15:44:42 +0000 Subject: Add latest changes from gitlab-org/gitlab@13-12-stable-ee --- .../base_after_export_strategy.rb | 2 +- .../after_export_strategies/web_upload_strategy.rb | 2 +- .../import_export/after_export_strategy_builder.rb | 2 +- lib/gitlab/import_export/attributes_finder.rb | 6 +- lib/gitlab/import_export/avatar_restorer.rb | 2 +- lib/gitlab/import_export/avatar_saver.rb | 2 +- lib/gitlab/import_export/base/relation_factory.rb | 3 +- lib/gitlab/import_export/command_line_util.rb | 13 ++++ .../decompressed_archive_size_validator.rb | 2 +- lib/gitlab/import_export/error.rb | 12 ++- lib/gitlab/import_export/file_importer.rb | 18 ++++- lib/gitlab/import_export/group/import_export.yml | 2 + .../import_export/group/legacy_import_export.yml | 2 + .../import_export/group/legacy_tree_restorer.rb | 2 +- .../import_export/group/legacy_tree_saver.rb | 4 +- lib/gitlab/import_export/group/tree_restorer.rb | 4 +- lib/gitlab/import_export/group/tree_saver.rb | 2 +- lib/gitlab/import_export/importer.rb | 6 +- lib/gitlab/import_export/json/legacy_reader.rb | 4 +- .../import_export/json/streaming_serializer.rb | 88 +++++++++++++++++----- lib/gitlab/import_export/lfs_restorer.rb | 6 +- lib/gitlab/import_export/lfs_saver.rb | 2 +- lib/gitlab/import_export/members_mapper.rb | 2 +- lib/gitlab/import_export/merge_request_parser.rb | 2 +- lib/gitlab/import_export/project/import_export.yml | 29 +++++++ .../import_export/project/relation_factory.rb | 19 ++++- lib/gitlab/import_export/project/tree_restorer.rb | 2 +- lib/gitlab/import_export/project/tree_saver.rb | 2 +- lib/gitlab/import_export/reader.rb | 2 +- lib/gitlab/import_export/relation_tree_restorer.rb | 9 ++- lib/gitlab/import_export/repo_restorer.rb | 4 +- lib/gitlab/import_export/repo_saver.rb | 2 +- lib/gitlab/import_export/saver.rb | 2 +- lib/gitlab/import_export/shared.rb | 2 +- lib/gitlab/import_export/snippet_repo_restorer.rb | 2 +- lib/gitlab/import_export/statistics_restorer.rb | 2 +- lib/gitlab/import_export/uploads_manager.rb | 4 +- lib/gitlab/import_export/uploads_restorer.rb | 2 +- lib/gitlab/import_export/uploads_saver.rb | 2 +- lib/gitlab/import_export/version_checker.rb | 8 +- lib/gitlab/import_export/version_saver.rb | 2 +- lib/gitlab/import_export/wiki_repo_saver.rb | 2 +- 42 files changed, 213 insertions(+), 74 deletions(-) (limited to 'lib/gitlab/import_export') diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb index b30258123d4..b43d0a0c3eb 100644 --- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb @@ -42,7 +42,7 @@ module Gitlab strategy_execute true - rescue => e + rescue StandardError => e project.import_export_shared.error(e) false ensure diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb index e2dba831661..1e8009d29c2 100644 --- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb +++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb @@ -28,7 +28,7 @@ module Gitlab def handle_response_error(response) unless response.success? - raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}") + raise StrategyError, "Error uploading the project. Code #{response.code}: #{response.message}" end end diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb index 37394f46a99..d7b30f46903 100644 --- a/lib/gitlab/import_export/after_export_strategy_builder.rb +++ b/lib/gitlab/import_export/after_export_strategy_builder.rb @@ -12,7 +12,7 @@ module Gitlab klass = strategy_klass.constantize rescue nil unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy - raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found") + raise StrategyNotFoundError, "Strategy #{strategy_klass} not found" end klass.new(**attributes.symbolize_keys) diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 1e98595bb07..4abc3da1190 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,7 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders def initialize(config:) @tree = config[:tree] || {} @@ -11,6 +11,7 @@ module Gitlab @excluded_attributes = config[:excluded_attributes] || {} @methods = config[:methods] || {} @preloads = config[:preloads] || {} + @export_reorders = config[:export_reorders] || {} end def find_root(model_key) @@ -33,7 +34,8 @@ module Gitlab except: @excluded_attributes[model_key], methods: @methods[model_key], include: resolve_model_tree(model_tree), - preload: resolve_preloads(model_key, model_tree) + preload: resolve_preloads(model_key, model_tree), + export_reorder: @export_reorders[model_key] }.compact end diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb index be1b97bd7a7..01ff99798d5 100644 --- a/lib/gitlab/import_export/avatar_restorer.rb +++ b/lib/gitlab/import_export/avatar_restorer.rb @@ -13,7 +13,7 @@ module Gitlab @project.avatar = File.open(avatar_export_file) @project.save! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb index 47ca898c690..7534ab5a9ce 100644 --- a/lib/gitlab/import_export/avatar_saver.rb +++ b/lib/gitlab/import_export/avatar_saver.rb @@ -16,7 +16,7 @@ module Gitlab shared: @shared, relative_export_path: 'avatar' ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 05a4a8f4c93..959ece4b903 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -44,8 +44,9 @@ module Gitlab relation_name.to_s.constantize end - def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) + def initialize(relation_sym:, relation_index:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: []) @relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym + @relation_index = relation_index @relation_hash = relation_hash.except('noteable_id') @members_mapper = members_mapper @object_builder = object_builder diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 2f8769e261d..ace9d83dc9a 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -14,6 +14,19 @@ module Gitlab untar_with_options(archive: archive, dir: dir, options: 'zxf') end + def gzip(dir:, filename:) + filepath = File.join(dir, filename) + cmd = %W(gzip #{filepath}) + + _, status = Gitlab::Popen.popen(cmd) + + if status == 0 + status + else + raise Gitlab::ImportExport::Error.file_compression_error + end + end + def mkdir_p(path) FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE) FileUtils.chmod(DEFAULT_DIR_MODE, path) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 37f1bdc3009..2baf2c61f7c 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -61,7 +61,7 @@ module Gitlab Process.kill(-1, pgrp) if pgrp false - rescue => e + rescue StandardError => e log_error(e.message) Process.kill(-1, pgrp) if pgrp diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index f11b7a0a298..4af6b03fe94 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -3,12 +3,20 @@ module Gitlab module ImportExport class Error < StandardError - def self.permission_error(user, importable) + def self.permission_error(user, object) self.new( "User with ID: %s does not have required permissions for %s: %s with ID: %s" % - [user.id, importable.class.name, importable.name, importable.id] + [user.id, object.class.name, object.name, object.id] ) end + + def self.unsupported_object_type_error + self.new('Unknown object type') + end + + def self.file_compression_error + self.new('File compression failed') + end end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 51d58aae54f..4b3258f8caa 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -33,7 +33,7 @@ module Gitlab validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size) decompress_archive end - rescue => e + rescue StandardError => e @shared.error(e) false ensure @@ -57,7 +57,7 @@ module Gitlab def decompress_archive result = untar_zxf(archive: @archive_file, dir: @shared.export_path) - raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result + raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result result end @@ -67,7 +67,17 @@ module Gitlab @archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable)) - download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file) + remote_download_or_download_or_copy_upload + end + + def remote_download_or_download_or_copy_upload + import_export_upload = @importable.import_export_upload + + if import_export_upload.remote_import_url.present? + download(import_export_upload.remote_import_url, @archive_file) + else + download_or_copy_upload(import_export_upload.import_file, @archive_file) + end end def remove_symlinks @@ -87,7 +97,7 @@ module Gitlab end def validate_decompressed_archive_size - raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid? + raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end def size_validator diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index e30206dc509..aceb4821a06 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -58,6 +58,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 5008639077c..19611e1b010 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -60,6 +60,8 @@ methods: preloads: +export_reorders: + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 5499b79cee6..2b95c098b59 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -45,7 +45,7 @@ module Gitlab return false if @shared.errors.any? true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb index 7ab81c09885..0f74fabeac3 100644 --- a/lib/gitlab/import_export/group/legacy_tree_saver.rb +++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb @@ -19,7 +19,7 @@ module Gitlab tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename) true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -35,7 +35,7 @@ module Gitlab end group_tree - rescue => e + rescue StandardError => e @shared.error(e) end diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb index 925ab6680ba..ea7de4cc896 100644 --- a/lib/gitlab/import_export/group/tree_restorer.rb +++ b/lib/gitlab/import_export/group/tree_restorer.rb @@ -26,7 +26,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -74,7 +74,7 @@ module Gitlab group = create_group(group_attributes) restore_group(group, group_attributes) - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_child', relation_key: 'group', diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index d538de33c51..0f588a55f9d 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -25,7 +25,7 @@ module Gitlab json_writer.write_relation_array('groups', '_all', all_groups) true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb index 390909efe36..c2510bbe938 100644 --- a/lib/gitlab/import_export/importer.rb +++ b/lib/gitlab/import_export/importer.rb @@ -21,15 +21,15 @@ module Gitlab if import_file && check_version! && restorers.all?(&:restore) && overwrite_project project else - raise Projects::ImportService::Error.new(shared.errors.to_sentence) + raise Projects::ImportService::Error, shared.errors.to_sentence end - rescue => e + rescue StandardError => e # If some exception was raised could mean that the SnippetsRepoRestorer # was not called. This would leave us with snippets without a repository. # This is a state we don't want them to be, so we better delete them. remove_non_migrated_snippets - raise Projects::ImportService::Error.new(e.message) + raise Projects::ImportService::Error, e.message ensure remove_base_tmp_dir remove_import_file diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index 12d6458aedc..f29c0a44188 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -28,9 +28,9 @@ module Gitlab def read_hash ActiveSupport::JSON.decode(IO.read(@path)) - rescue => e + rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 05b7679e0ff..ec42c5e51c0 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -38,16 +38,6 @@ module Gitlab end end - private - - attr_reader :json_writer, :relations_schema, :exportable - - def serialize_root - attributes = exportable.as_json( - relations_schema.merge(include: nil, preloads: nil)) - json_writer.write_attributes(@exportable_path, attributes) - end - def serialize_relation(definition) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? @@ -64,17 +54,22 @@ module Gitlab end end + private + + attr_reader :json_writer, :relations_schema, :exportable + + def serialize_root + attributes = exportable.as_json( + relations_schema.merge(include: nil, preloads: nil)) + json_writer.write_attributes(@exportable_path, attributes) + end + def serialize_many_relations(key, records, options) enumerator = Enumerator.new do |items| key_preloads = preloads&.dig(key) - records = records.preload(key_preloads) if key_preloads - records.in_batches(of: batch_size) do |batch| # rubocop:disable Cop/InBatches - # order each batch by its primary key to ensure - # consistent and predictable ordering of each exported relation - # as additional `WHERE` clauses can impact the order in which data is being - # returned by database when no `ORDER` is specified - batch = batch.reorder(batch.klass.primary_key) + batch(records, key) do |batch| + batch = batch.preload(key_preloads) if key_preloads batch.each do |record| items << Raw.new(record.to_json(options)) @@ -85,6 +80,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def batch(relation, key) + opts = { of: batch_size } + order_by = reorders(relation, key) + + # we need to sort issues by non primary key column(relative_position) + # and `in_batches` does not support that + if order_by + scope = relation.reorder(order_by) + + Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true).each_batch(**opts) do |batch| + yield batch + end + else + relation.in_batches(**opts) do |batch| # rubocop:disable Cop/InBatches + # order each batch by its primary key to ensure + # consistent and predictable ordering of each exported relation + # as additional `WHERE` clauses can impact the order in which data is being + # returned by database when no `ORDER` is specified + yield batch.reorder(batch.klass.primary_key) + end + end + end + def serialize_many_each(key, records, options) enumerator = Enumerator.new do |items| records.each do |record| @@ -112,6 +130,42 @@ module Gitlab def batch_size @batch_size ||= self.class.batch_size(@exportable) end + + def reorders(relation, key) + export_reorder = relations_schema[:export_reorder]&.dig(key) + return unless export_reorder + + custom_reorder(relation.klass, export_reorder) + end + + def custom_reorder(klass, order_by) + arel_table = klass.arel_table + column = order_by[:column] || klass.primary_key + direction = order_by[:direction] || :asc + nulls_position = order_by[:nulls_position] || :nulls_last + + arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert + reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction] + reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position] + order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position) + reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position) + + ::Gitlab::Pagination::Keyset::Order.build([ + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: column, + column_expression: arel_table[column], + order_expression: order_expression, + reversed_order_expression: reverse_order_expression, + order_direction: direction, + nullable: nulls_position, + distinct: false + ), + ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new( + attribute_name: klass.primary_key, + order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym]) + ) + ]) + end end end end diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index ef83cdf24b1..d73ae1410a3 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -20,7 +20,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -73,8 +73,8 @@ module Gitlab begin json = IO.read(lfs_json_path) ActiveSupport::JSON.decode(json) - rescue - raise Gitlab::ImportExport::Error.new('Incorrect JSON format') + rescue StandardError + raise Gitlab::ImportExport::Error, 'Incorrect JSON format' end end diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb index 4964b8b16f4..47acd49d529 100644 --- a/lib/gitlab/import_export/lfs_saver.rb +++ b/lib/gitlab/import_export/lfs_saver.rb @@ -27,7 +27,7 @@ module Gitlab write_lfs_json true - rescue => e + rescue StandardError => e shared.error(e) false diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index 6b37683ea68..ff972cf9352 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -52,7 +52,7 @@ module Gitlab @importable.members.destroy_all # rubocop: disable Cop/DestroyAll relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true) - rescue => e + rescue StandardError => e raise e, "Error adding importer user to #{@importable.class} members. #{e.message}" end diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb index 4643742b607..3910afef108 100644 --- a/lib/gitlab/import_export/merge_request_parser.rb +++ b/lib/gitlab/import_export/merge_request_parser.rb @@ -40,7 +40,7 @@ module Gitlab # the commits are missing. def create_source_branch @project.repository.create_branch(@merge_request.source_branch, @diff_head_sha) - rescue => err + rescue StandardError => err Gitlab::Import::Logger.warn( message: 'Import warning: Failed to create source branch', source_branch: @merge_request.source_branch, diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 42d32593cbd..d000c331b6d 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -132,6 +132,7 @@ excluded_attributes: - :avatar - :import_type - :import_source + - :integrations - :mirror - :runners_token - :runners_token_encrypted @@ -152,6 +153,7 @@ excluded_attributes: - :bfg_object_map - :detected_repository_languages - :tag_list + - :topic_list - :mirror_user_id - :mirror_trigger_builds - :only_mirror_protected_branches @@ -261,6 +263,8 @@ excluded_attributes: - :resource_group_id - :waiting_for_resource_at - :processed + - :id_convert_to_bigint + - :stage_id_convert_to_bigint sentry_issue: - :issue_id push_event_payload: @@ -393,6 +397,8 @@ methods: - :state preloads: + issues: + project: :route statuses: # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` # tags: # needed by tag_list @@ -402,6 +408,29 @@ preloads: target_project: # needed by target_branch_sha assignees: # needed by assigne_id that is implemented by DeprecatedAssignee +# Specify a custom export reordering for a given relationship +# For example for issues we use a custom export reordering by relative_position, so that on import, we can reset the +# relative position value, but still keep the issues order to the order in which issues were in the exported project. +# By default the ordering of relations is done by PK. +# column - specify the column by which to reorder, by default it is relation's PK +# direction - specify the ordering direction :asc or :desc, default :asc +# nulls_position - specify where would null values be positioned. Because custom ordering column can contain nulls we +# need to also specify where would the nulls be placed. It can be :nulls_last or :nulls_first, defaults +# to :nulls_last +# Example: +# export_reorders: +# project: +# issues: +# column: :relative_position +# direction: :asc +# nulls_position: :nulls_last +export_reorders: + project: + issues: + column: :relative_position + direction: :asc + nulls_position: :nulls_last + # EE specific relationships and settings to include. All of this will be merged # into the previous structures if EE is used. ee: diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index ae92228276e..4678396f97e 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -80,6 +80,7 @@ module Gitlab when :notes then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build + when :issues then setup_issue end update_project_references @@ -135,6 +136,22 @@ module Gitlab end end + def setup_issue + @relation_hash['relative_position'] = compute_relative_position + end + + def compute_relative_position + return unless max_relative_position + + max_relative_position + (@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE + end + + def max_relative_position + Rails.cache.fetch("import:#{@importable.model_name.plural}:#{@importable.id}:hierarchy_max_issues_relative_position", expires_in: 24.hours) do + ::RelativePositioning.mover.context(Issue.in_projects(@importable.root_ancestor.all_projects).first)&.max_relative_position || ::Gitlab::RelativePositioning::START_POSITION + end + end + def legacy_trigger? @relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil? end @@ -158,4 +175,4 @@ module Gitlab end end -Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory') +Gitlab::ImportExport::Project::RelationFactory.prepend_mod_with('Gitlab::ImportExport::Project::RelationFactory') diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index fb9e5be1877..113502b4e3c 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -39,7 +39,7 @@ module Gitlab else false end - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 80dacf2eb20..16012f3c0c0 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -22,7 +22,7 @@ module Gitlab ).execute true - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb index 8d36d05ca6f..b9a1aee3b8e 100644 --- a/lib/gitlab/import_export/reader.rb +++ b/lib/gitlab/import_export/reader.rb @@ -35,7 +35,7 @@ module Gitlab def tree_by_key(key) attributes_finder.find_root(key) - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb index 8bc87ecb071..46b82240ef7 100644 --- a/lib/gitlab/import_export/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/relation_tree_restorer.rb @@ -48,7 +48,7 @@ module Gitlab @importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -81,7 +81,7 @@ module Gitlab relation_object.save! log_relation_creation(@importable, relation_key, relation_object) end - rescue => e + rescue StandardError => e import_failure_service.log_import_failure( source: 'process_relation_item!', relation_key: relation_key, @@ -155,7 +155,7 @@ module Gitlab transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index) end - relation = @relation_factory.create(**relation_factory_params(relation_key, data_hash)) + relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash)) if relation && !relation.valid? @shared.logger.warn( @@ -221,8 +221,9 @@ module Gitlab importable_class.to_s.downcase.to_sym end - def relation_factory_params(relation_key, data_hash) + def relation_factory_params(relation_key, relation_index, data_hash) { + relation_index: relation_index, relation_sym: relation_key.to_sym, relation_hash: data_hash, importable: @importable, diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 998da3e4afb..1c6629cf942 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -22,7 +22,7 @@ module Gitlab update_importable_repository_info true - rescue => e + rescue StandardError => e shared.error(e) false end @@ -52,4 +52,4 @@ module Gitlab end end -Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer') +Gitlab::ImportExport::RepoRestorer.prepend_mod_with('Gitlab::ImportExport::RepoRestorer') diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb index 0fdd0722b65..fae07039139 100644 --- a/lib/gitlab/import_export/repo_saver.rb +++ b/lib/gitlab/import_export/repo_saver.rb @@ -40,7 +40,7 @@ module Gitlab mkdir_p(File.dirname(bundle_full_path)) repository.bundle_to_disk(bundle_full_path) - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb index bb2bbda4bd6..bec709f4a36 100644 --- a/lib/gitlab/import_export/saver.rb +++ b/lib/gitlab/import_export/saver.rb @@ -27,7 +27,7 @@ module Gitlab @shared.error(Gitlab::ImportExport::Error.new(error_message)) false end - rescue => e + rescue StandardError => e @shared.error(e) false ensure diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb index 09ed4eb568d..f295ab38de0 100644 --- a/lib/gitlab/import_export/shared.rb +++ b/lib/gitlab/import_export/shared.rb @@ -90,7 +90,7 @@ module Gitlab when 'Group' @exportable.full_path else - raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}") + raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}" end end diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb index 2d0aa05fc3c..cb13972f8f2 100644 --- a/lib/gitlab/import_export/snippet_repo_restorer.rb +++ b/lib/gitlab/import_export/snippet_repo_restorer.rb @@ -23,7 +23,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e shared.error(e) false end diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb index 3fafb01c37c..a3ad5edc2cc 100644 --- a/lib/gitlab/import_export/statistics_restorer.rb +++ b/lib/gitlab/import_export/statistics_restorer.rb @@ -10,7 +10,7 @@ module Gitlab def restore @project.statistics.refresh! - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb index 2f15cdd7506..ad19508fb99 100644 --- a/lib/gitlab/import_export/uploads_manager.rb +++ b/lib/gitlab/import_export/uploads_manager.rb @@ -17,7 +17,7 @@ module Gitlab copy_project_uploads true - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -30,7 +30,7 @@ module Gitlab end true - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb index 5f422dcbefa..741c6555aad 100644 --- a/lib/gitlab/import_export/uploads_restorer.rb +++ b/lib/gitlab/import_export/uploads_restorer.rb @@ -8,7 +8,7 @@ module Gitlab project: @project, shared: @shared ).restore - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb index be1066c30b2..9f58609fa17 100644 --- a/lib/gitlab/import_export/uploads_saver.rb +++ b/lib/gitlab/import_export/uploads_saver.rb @@ -13,7 +13,7 @@ module Gitlab project: @project, shared: @shared ).save - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb index 48f5b558e52..5ec9db00d0a 100644 --- a/lib/gitlab/import_export/version_checker.rb +++ b/lib/gitlab/import_export/version_checker.rb @@ -14,7 +14,7 @@ module Gitlab def check! version = File.open(version_file, &:readline) verify_version!(version) - rescue => e + rescue StandardError => e @shared.error(e) false end @@ -27,7 +27,7 @@ module Gitlab def verify_version!(version) if different_version?(version) - raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}") + raise Gitlab::ImportExport::Error, "Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}" else true end @@ -35,13 +35,13 @@ module Gitlab def different_version?(version) Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version) - rescue => e + rescue StandardError => e Gitlab::Import::Logger.error( message: 'Import error', error: e.message ) - raise Gitlab::ImportExport::Error.new('Incorrect VERSION format') + raise Gitlab::ImportExport::Error, 'Incorrect VERSION format' end end end diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb index dab8bbf539d..e8f68f93af0 100644 --- a/lib/gitlab/import_export/version_saver.rb +++ b/lib/gitlab/import_export/version_saver.rb @@ -15,7 +15,7 @@ module Gitlab File.write(version_file, Gitlab::ImportExport.version, mode: 'w') File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w') File.write(gitlab_revision_file, Gitlab.revision, mode: 'w') - rescue => e + rescue StandardError => e @shared.error(e) false end diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb index 4b1cf4915e4..162b474bfeb 100644 --- a/lib/gitlab/import_export/wiki_repo_saver.rb +++ b/lib/gitlab/import_export/wiki_repo_saver.rb @@ -20,4 +20,4 @@ module Gitlab end end -Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver') +Gitlab::ImportExport::WikiRepoSaver.prepend_mod_with('Gitlab::ImportExport::WikiRepoSaver') -- cgit v1.2.3