Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-05-19 18:44:42 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2021-05-19 18:44:42 +0300
commit4555e1b21c365ed8303ffb7a3325d773c9b8bf31 (patch)
tree5423a1c7516cffe36384133ade12572cf709398d /lib/gitlab/import_export
parente570267f2f6b326480d284e0164a6464ba4081bc (diff)
Add latest changes from gitlab-org/gitlab@13-12-stable-eev13.12.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r--lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb2
-rw-r--r--lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb2
-rw-r--r--lib/gitlab/import_export/after_export_strategy_builder.rb2
-rw-r--r--lib/gitlab/import_export/attributes_finder.rb6
-rw-r--r--lib/gitlab/import_export/avatar_restorer.rb2
-rw-r--r--lib/gitlab/import_export/avatar_saver.rb2
-rw-r--r--lib/gitlab/import_export/base/relation_factory.rb3
-rw-r--r--lib/gitlab/import_export/command_line_util.rb13
-rw-r--r--lib/gitlab/import_export/decompressed_archive_size_validator.rb2
-rw-r--r--lib/gitlab/import_export/error.rb12
-rw-r--r--lib/gitlab/import_export/file_importer.rb18
-rw-r--r--lib/gitlab/import_export/group/import_export.yml2
-rw-r--r--lib/gitlab/import_export/group/legacy_import_export.yml2
-rw-r--r--lib/gitlab/import_export/group/legacy_tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/group/legacy_tree_saver.rb4
-rw-r--r--lib/gitlab/import_export/group/tree_restorer.rb4
-rw-r--r--lib/gitlab/import_export/group/tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/importer.rb6
-rw-r--r--lib/gitlab/import_export/json/legacy_reader.rb4
-rw-r--r--lib/gitlab/import_export/json/streaming_serializer.rb88
-rw-r--r--lib/gitlab/import_export/lfs_restorer.rb6
-rw-r--r--lib/gitlab/import_export/lfs_saver.rb2
-rw-r--r--lib/gitlab/import_export/members_mapper.rb2
-rw-r--r--lib/gitlab/import_export/merge_request_parser.rb2
-rw-r--r--lib/gitlab/import_export/project/import_export.yml29
-rw-r--r--lib/gitlab/import_export/project/relation_factory.rb19
-rw-r--r--lib/gitlab/import_export/project/tree_restorer.rb2
-rw-r--r--lib/gitlab/import_export/project/tree_saver.rb2
-rw-r--r--lib/gitlab/import_export/reader.rb2
-rw-r--r--lib/gitlab/import_export/relation_tree_restorer.rb9
-rw-r--r--lib/gitlab/import_export/repo_restorer.rb4
-rw-r--r--lib/gitlab/import_export/repo_saver.rb2
-rw-r--r--lib/gitlab/import_export/saver.rb2
-rw-r--r--lib/gitlab/import_export/shared.rb2
-rw-r--r--lib/gitlab/import_export/snippet_repo_restorer.rb2
-rw-r--r--lib/gitlab/import_export/statistics_restorer.rb2
-rw-r--r--lib/gitlab/import_export/uploads_manager.rb4
-rw-r--r--lib/gitlab/import_export/uploads_restorer.rb2
-rw-r--r--lib/gitlab/import_export/uploads_saver.rb2
-rw-r--r--lib/gitlab/import_export/version_checker.rb8
-rw-r--r--lib/gitlab/import_export/version_saver.rb2
-rw-r--r--lib/gitlab/import_export/wiki_repo_saver.rb2
42 files changed, 213 insertions, 74 deletions
diff --git a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
index b30258123d4..b43d0a0c3eb 100644
--- a/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/base_after_export_strategy.rb
@@ -42,7 +42,7 @@ module Gitlab
strategy_execute
true
- rescue => e
+ rescue StandardError => e
project.import_export_shared.error(e)
false
ensure
diff --git a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
index e2dba831661..1e8009d29c2 100644
--- a/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
+++ b/lib/gitlab/import_export/after_export_strategies/web_upload_strategy.rb
@@ -28,7 +28,7 @@ module Gitlab
def handle_response_error(response)
unless response.success?
- raise StrategyError.new("Error uploading the project. Code #{response.code}: #{response.message}")
+ raise StrategyError, "Error uploading the project. Code #{response.code}: #{response.message}"
end
end
diff --git a/lib/gitlab/import_export/after_export_strategy_builder.rb b/lib/gitlab/import_export/after_export_strategy_builder.rb
index 37394f46a99..d7b30f46903 100644
--- a/lib/gitlab/import_export/after_export_strategy_builder.rb
+++ b/lib/gitlab/import_export/after_export_strategy_builder.rb
@@ -12,7 +12,7 @@ module Gitlab
klass = strategy_klass.constantize rescue nil
unless klass && klass < AfterExportStrategies::BaseAfterExportStrategy
- raise StrategyNotFoundError.new("Strategy #{strategy_klass} not found")
+ raise StrategyNotFoundError, "Strategy #{strategy_klass} not found"
end
klass.new(**attributes.symbolize_keys)
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index 1e98595bb07..4abc3da1190 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -3,7 +3,7 @@
module Gitlab
module ImportExport
class AttributesFinder
- attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads
+ attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders
def initialize(config:)
@tree = config[:tree] || {}
@@ -11,6 +11,7 @@ module Gitlab
@excluded_attributes = config[:excluded_attributes] || {}
@methods = config[:methods] || {}
@preloads = config[:preloads] || {}
+ @export_reorders = config[:export_reorders] || {}
end
def find_root(model_key)
@@ -33,7 +34,8 @@ module Gitlab
except: @excluded_attributes[model_key],
methods: @methods[model_key],
include: resolve_model_tree(model_tree),
- preload: resolve_preloads(model_key, model_tree)
+ preload: resolve_preloads(model_key, model_tree),
+ export_reorder: @export_reorders[model_key]
}.compact
end
diff --git a/lib/gitlab/import_export/avatar_restorer.rb b/lib/gitlab/import_export/avatar_restorer.rb
index be1b97bd7a7..01ff99798d5 100644
--- a/lib/gitlab/import_export/avatar_restorer.rb
+++ b/lib/gitlab/import_export/avatar_restorer.rb
@@ -13,7 +13,7 @@ module Gitlab
@project.avatar = File.open(avatar_export_file)
@project.save!
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/avatar_saver.rb b/lib/gitlab/import_export/avatar_saver.rb
index 47ca898c690..7534ab5a9ce 100644
--- a/lib/gitlab/import_export/avatar_saver.rb
+++ b/lib/gitlab/import_export/avatar_saver.rb
@@ -16,7 +16,7 @@ module Gitlab
shared: @shared,
relative_export_path: 'avatar'
).save
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index 05a4a8f4c93..959ece4b903 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -44,8 +44,9 @@ module Gitlab
relation_name.to_s.constantize
end
- def initialize(relation_sym:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: [])
+ def initialize(relation_sym:, relation_index:, relation_hash:, members_mapper:, object_builder:, user:, importable:, excluded_keys: [])
@relation_name = self.class.overrides[relation_sym]&.to_sym || relation_sym
+ @relation_index = relation_index
@relation_hash = relation_hash.except('noteable_id')
@members_mapper = members_mapper
@object_builder = object_builder
diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb
index 2f8769e261d..ace9d83dc9a 100644
--- a/lib/gitlab/import_export/command_line_util.rb
+++ b/lib/gitlab/import_export/command_line_util.rb
@@ -14,6 +14,19 @@ module Gitlab
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
+ def gzip(dir:, filename:)
+ filepath = File.join(dir, filename)
+ cmd = %W(gzip #{filepath})
+
+ _, status = Gitlab::Popen.popen(cmd)
+
+ if status == 0
+ status
+ else
+ raise Gitlab::ImportExport::Error.file_compression_error
+ end
+ end
+
def mkdir_p(path)
FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE)
FileUtils.chmod(DEFAULT_DIR_MODE, path)
diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
index 37f1bdc3009..2baf2c61f7c 100644
--- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb
+++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb
@@ -61,7 +61,7 @@ module Gitlab
Process.kill(-1, pgrp) if pgrp
false
- rescue => e
+ rescue StandardError => e
log_error(e.message)
Process.kill(-1, pgrp) if pgrp
diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb
index f11b7a0a298..4af6b03fe94 100644
--- a/lib/gitlab/import_export/error.rb
+++ b/lib/gitlab/import_export/error.rb
@@ -3,12 +3,20 @@
module Gitlab
module ImportExport
class Error < StandardError
- def self.permission_error(user, importable)
+ def self.permission_error(user, object)
self.new(
"User with ID: %s does not have required permissions for %s: %s with ID: %s" %
- [user.id, importable.class.name, importable.name, importable.id]
+ [user.id, object.class.name, object.name, object.id]
)
end
+
+ def self.unsupported_object_type_error
+ self.new('Unknown object type')
+ end
+
+ def self.file_compression_error
+ self.new('File compression failed')
+ end
end
end
end
diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb
index 51d58aae54f..4b3258f8caa 100644
--- a/lib/gitlab/import_export/file_importer.rb
+++ b/lib/gitlab/import_export/file_importer.rb
@@ -33,7 +33,7 @@ module Gitlab
validate_decompressed_archive_size if Feature.enabled?(:validate_import_decompressed_archive_size)
decompress_archive
end
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
ensure
@@ -57,7 +57,7 @@ module Gitlab
def decompress_archive
result = untar_zxf(archive: @archive_file, dir: @shared.export_path)
- raise ImporterError.new("Unable to decompress #{@archive_file} into #{@shared.export_path}") unless result
+ raise ImporterError, "Unable to decompress #{@archive_file} into #{@shared.export_path}" unless result
result
end
@@ -67,7 +67,17 @@ module Gitlab
@archive_file = File.join(@shared.archive_path, Gitlab::ImportExport.export_filename(exportable: @importable))
- download_or_copy_upload(@importable.import_export_upload.import_file, @archive_file)
+ remote_download_or_download_or_copy_upload
+ end
+
+ def remote_download_or_download_or_copy_upload
+ import_export_upload = @importable.import_export_upload
+
+ if import_export_upload.remote_import_url.present?
+ download(import_export_upload.remote_import_url, @archive_file)
+ else
+ download_or_copy_upload(import_export_upload.import_file, @archive_file)
+ end
end
def remove_symlinks
@@ -87,7 +97,7 @@ module Gitlab
end
def validate_decompressed_archive_size
- raise ImporterError.new(_('Decompressed archive size validation failed.')) unless size_validator.valid?
+ raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid?
end
def size_validator
diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml
index e30206dc509..aceb4821a06 100644
--- a/lib/gitlab/import_export/group/import_export.yml
+++ b/lib/gitlab/import_export/group/import_export.yml
@@ -58,6 +58,8 @@ methods:
preloads:
+export_reorders:
+
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee:
diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml
index 5008639077c..19611e1b010 100644
--- a/lib/gitlab/import_export/group/legacy_import_export.yml
+++ b/lib/gitlab/import_export/group/legacy_import_export.yml
@@ -60,6 +60,8 @@ methods:
preloads:
+export_reorders:
+
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee:
diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
index 5499b79cee6..2b95c098b59 100644
--- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
@@ -45,7 +45,7 @@ module Gitlab
return false if @shared.errors.any?
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/group/legacy_tree_saver.rb b/lib/gitlab/import_export/group/legacy_tree_saver.rb
index 7ab81c09885..0f74fabeac3 100644
--- a/lib/gitlab/import_export/group/legacy_tree_saver.rb
+++ b/lib/gitlab/import_export/group/legacy_tree_saver.rb
@@ -19,7 +19,7 @@ module Gitlab
tree_saver.save(group_tree, @shared.export_path, ImportExport.group_filename)
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
@@ -35,7 +35,7 @@ module Gitlab
end
group_tree
- rescue => e
+ rescue StandardError => e
@shared.error(e)
end
diff --git a/lib/gitlab/import_export/group/tree_restorer.rb b/lib/gitlab/import_export/group/tree_restorer.rb
index 925ab6680ba..ea7de4cc896 100644
--- a/lib/gitlab/import_export/group/tree_restorer.rb
+++ b/lib/gitlab/import_export/group/tree_restorer.rb
@@ -26,7 +26,7 @@ module Gitlab
end
true
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
end
@@ -74,7 +74,7 @@ module Gitlab
group = create_group(group_attributes)
restore_group(group, group_attributes)
- rescue => e
+ rescue StandardError => e
import_failure_service.log_import_failure(
source: 'process_child',
relation_key: 'group',
diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
index d538de33c51..0f588a55f9d 100644
--- a/lib/gitlab/import_export/group/tree_saver.rb
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -25,7 +25,7 @@ module Gitlab
json_writer.write_relation_array('groups', '_all', all_groups)
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
ensure
diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index 390909efe36..c2510bbe938 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -21,15 +21,15 @@ module Gitlab
if import_file && check_version! && restorers.all?(&:restore) && overwrite_project
project
else
- raise Projects::ImportService::Error.new(shared.errors.to_sentence)
+ raise Projects::ImportService::Error, shared.errors.to_sentence
end
- rescue => e
+ rescue StandardError => e
# If some exception was raised could mean that the SnippetsRepoRestorer
# was not called. This would leave us with snippets without a repository.
# This is a state we don't want them to be, so we better delete them.
remove_non_migrated_snippets
- raise Projects::ImportService::Error.new(e.message)
+ raise Projects::ImportService::Error, e.message
ensure
remove_base_tmp_dir
remove_import_file
diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb
index 12d6458aedc..f29c0a44188 100644
--- a/lib/gitlab/import_export/json/legacy_reader.rb
+++ b/lib/gitlab/import_export/json/legacy_reader.rb
@@ -28,9 +28,9 @@ module Gitlab
def read_hash
ActiveSupport::JSON.decode(IO.read(@path))
- rescue => e
+ rescue StandardError => e
Gitlab::ErrorTracking.log_exception(e)
- raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+ raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
end
end
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index 05b7679e0ff..ec42c5e51c0 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -38,16 +38,6 @@ module Gitlab
end
end
- private
-
- attr_reader :json_writer, :relations_schema, :exportable
-
- def serialize_root
- attributes = exportable.as_json(
- relations_schema.merge(include: nil, preloads: nil))
- json_writer.write_attributes(@exportable_path, attributes)
- end
-
def serialize_relation(definition)
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
@@ -64,17 +54,22 @@ module Gitlab
end
end
+ private
+
+ attr_reader :json_writer, :relations_schema, :exportable
+
+ def serialize_root
+ attributes = exportable.as_json(
+ relations_schema.merge(include: nil, preloads: nil))
+ json_writer.write_attributes(@exportable_path, attributes)
+ end
+
def serialize_many_relations(key, records, options)
enumerator = Enumerator.new do |items|
key_preloads = preloads&.dig(key)
- records = records.preload(key_preloads) if key_preloads
- records.in_batches(of: batch_size) do |batch| # rubocop:disable Cop/InBatches
- # order each batch by its primary key to ensure
- # consistent and predictable ordering of each exported relation
- # as additional `WHERE` clauses can impact the order in which data is being
- # returned by database when no `ORDER` is specified
- batch = batch.reorder(batch.klass.primary_key)
+ batch(records, key) do |batch|
+ batch = batch.preload(key_preloads) if key_preloads
batch.each do |record|
items << Raw.new(record.to_json(options))
@@ -85,6 +80,29 @@ module Gitlab
json_writer.write_relation_array(@exportable_path, key, enumerator)
end
+ def batch(relation, key)
+ opts = { of: batch_size }
+ order_by = reorders(relation, key)
+
+ # we need to sort issues by non primary key column(relative_position)
+ # and `in_batches` does not support that
+ if order_by
+ scope = relation.reorder(order_by)
+
+ Gitlab::Pagination::Keyset::Iterator.new(scope: scope, use_union_optimization: true).each_batch(**opts) do |batch|
+ yield batch
+ end
+ else
+ relation.in_batches(**opts) do |batch| # rubocop:disable Cop/InBatches
+ # order each batch by its primary key to ensure
+ # consistent and predictable ordering of each exported relation
+ # as additional `WHERE` clauses can impact the order in which data is being
+ # returned by database when no `ORDER` is specified
+ yield batch.reorder(batch.klass.primary_key)
+ end
+ end
+ end
+
def serialize_many_each(key, records, options)
enumerator = Enumerator.new do |items|
records.each do |record|
@@ -112,6 +130,42 @@ module Gitlab
def batch_size
@batch_size ||= self.class.batch_size(@exportable)
end
+
+ def reorders(relation, key)
+ export_reorder = relations_schema[:export_reorder]&.dig(key)
+ return unless export_reorder
+
+ custom_reorder(relation.klass, export_reorder)
+ end
+
+ def custom_reorder(klass, order_by)
+ arel_table = klass.arel_table
+ column = order_by[:column] || klass.primary_key
+ direction = order_by[:direction] || :asc
+ nulls_position = order_by[:nulls_position] || :nulls_last
+
+ arel_order_classes = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::AREL_ORDER_CLASSES.invert
+ reverse_direction = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_ORDER_DIRECTIONS[direction]
+ reverse_nulls_position = ::Gitlab::Pagination::Keyset::ColumnOrderDefinition::REVERSED_NULL_POSITIONS[nulls_position]
+ order_expression = ::Gitlab::Database.nulls_order(column, direction, nulls_position)
+ reverse_order_expression = ::Gitlab::Database.nulls_order(column, reverse_direction, reverse_nulls_position)
+
+ ::Gitlab::Pagination::Keyset::Order.build([
+ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+ attribute_name: column,
+ column_expression: arel_table[column],
+ order_expression: order_expression,
+ reversed_order_expression: reverse_order_expression,
+ order_direction: direction,
+ nullable: nulls_position,
+ distinct: false
+ ),
+ ::Gitlab::Pagination::Keyset::ColumnOrderDefinition.new(
+ attribute_name: klass.primary_key,
+ order_expression: arel_order_classes[direction].new(arel_table[klass.primary_key.to_sym])
+ )
+ ])
+ end
end
end
end
diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb
index ef83cdf24b1..d73ae1410a3 100644
--- a/lib/gitlab/import_export/lfs_restorer.rb
+++ b/lib/gitlab/import_export/lfs_restorer.rb
@@ -20,7 +20,7 @@ module Gitlab
end
true
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
end
@@ -73,8 +73,8 @@ module Gitlab
begin
json = IO.read(lfs_json_path)
ActiveSupport::JSON.decode(json)
- rescue
- raise Gitlab::ImportExport::Error.new('Incorrect JSON format')
+ rescue StandardError
+ raise Gitlab::ImportExport::Error, 'Incorrect JSON format'
end
end
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb
index 4964b8b16f4..47acd49d529 100644
--- a/lib/gitlab/import_export/lfs_saver.rb
+++ b/lib/gitlab/import_export/lfs_saver.rb
@@ -27,7 +27,7 @@ module Gitlab
write_lfs_json
true
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index 6b37683ea68..ff972cf9352 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -52,7 +52,7 @@ module Gitlab
@importable.members.destroy_all # rubocop: disable Cop/DestroyAll
relation_class.create!(user: @user, access_level: highest_access_level, source_id: @importable.id, importing: true)
- rescue => e
+ rescue StandardError => e
raise e, "Error adding importer user to #{@importable.class} members. #{e.message}"
end
diff --git a/lib/gitlab/import_export/merge_request_parser.rb b/lib/gitlab/import_export/merge_request_parser.rb
index 4643742b607..3910afef108 100644
--- a/lib/gitlab/import_export/merge_request_parser.rb
+++ b/lib/gitlab/import_export/merge_request_parser.rb
@@ -40,7 +40,7 @@ module Gitlab
# the commits are missing.
def create_source_branch
@project.repository.create_branch(@merge_request.source_branch, @diff_head_sha)
- rescue => err
+ rescue StandardError => err
Gitlab::Import::Logger.warn(
message: 'Import warning: Failed to create source branch',
source_branch: @merge_request.source_branch,
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 42d32593cbd..d000c331b6d 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -132,6 +132,7 @@ excluded_attributes:
- :avatar
- :import_type
- :import_source
+ - :integrations
- :mirror
- :runners_token
- :runners_token_encrypted
@@ -152,6 +153,7 @@ excluded_attributes:
- :bfg_object_map
- :detected_repository_languages
- :tag_list
+ - :topic_list
- :mirror_user_id
- :mirror_trigger_builds
- :only_mirror_protected_branches
@@ -261,6 +263,8 @@ excluded_attributes:
- :resource_group_id
- :waiting_for_resource_at
- :processed
+ - :id_convert_to_bigint
+ - :stage_id_convert_to_bigint
sentry_issue:
- :issue_id
push_event_payload:
@@ -393,6 +397,8 @@ methods:
- :state
preloads:
+ issues:
+ project: :route
statuses:
# TODO: We cannot preload tags, as they are not part of `GenericCommitStatus`
# tags: # needed by tag_list
@@ -402,6 +408,29 @@ preloads:
target_project: # needed by target_branch_sha
assignees: # needed by assigne_id that is implemented by DeprecatedAssignee
+# Specify a custom export reordering for a given relationship
+# For example for issues we use a custom export reordering by relative_position, so that on import, we can reset the
+# relative position value, but still keep the issues order to the order in which issues were in the exported project.
+# By default the ordering of relations is done by PK.
+# column - specify the column by which to reorder, by default it is relation's PK
+# direction - specify the ordering direction :asc or :desc, default :asc
+# nulls_position - specify where would null values be positioned. Because custom ordering column can contain nulls we
+# need to also specify where would the nulls be placed. It can be :nulls_last or :nulls_first, defaults
+# to :nulls_last
+# Example:
+# export_reorders:
+# project:
+# issues:
+# column: :relative_position
+# direction: :asc
+# nulls_position: :nulls_last
+export_reorders:
+ project:
+ issues:
+ column: :relative_position
+ direction: :asc
+ nulls_position: :nulls_last
+
# EE specific relationships and settings to include. All of this will be merged
# into the previous structures if EE is used.
ee:
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index ae92228276e..4678396f97e 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -80,6 +80,7 @@ module Gitlab
when :notes then setup_note
when :'Ci::Pipeline' then setup_pipeline
when *BUILD_MODELS then setup_build
+ when :issues then setup_issue
end
update_project_references
@@ -135,6 +136,22 @@ module Gitlab
end
end
+ def setup_issue
+ @relation_hash['relative_position'] = compute_relative_position
+ end
+
+ def compute_relative_position
+ return unless max_relative_position
+
+ max_relative_position + (@relation_index + 1) * Gitlab::RelativePositioning::IDEAL_DISTANCE
+ end
+
+ def max_relative_position
+ Rails.cache.fetch("import:#{@importable.model_name.plural}:#{@importable.id}:hierarchy_max_issues_relative_position", expires_in: 24.hours) do
+ ::RelativePositioning.mover.context(Issue.in_projects(@importable.root_ancestor.all_projects).first)&.max_relative_position || ::Gitlab::RelativePositioning::START_POSITION
+ end
+ end
+
def legacy_trigger?
@relation_name == :'Ci::Trigger' && @relation_hash['owner_id'].nil?
end
@@ -158,4 +175,4 @@ module Gitlab
end
end
-Gitlab::ImportExport::Project::RelationFactory.prepend_if_ee('::EE::Gitlab::ImportExport::Project::RelationFactory')
+Gitlab::ImportExport::Project::RelationFactory.prepend_mod_with('Gitlab::ImportExport::Project::RelationFactory')
diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb
index fb9e5be1877..113502b4e3c 100644
--- a/lib/gitlab/import_export/project/tree_restorer.rb
+++ b/lib/gitlab/import_export/project/tree_restorer.rb
@@ -39,7 +39,7 @@ module Gitlab
else
false
end
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 80dacf2eb20..16012f3c0c0 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -22,7 +22,7 @@ module Gitlab
).execute
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
ensure
diff --git a/lib/gitlab/import_export/reader.rb b/lib/gitlab/import_export/reader.rb
index 8d36d05ca6f..b9a1aee3b8e 100644
--- a/lib/gitlab/import_export/reader.rb
+++ b/lib/gitlab/import_export/reader.rb
@@ -35,7 +35,7 @@ module Gitlab
def tree_by_key(key)
attributes_finder.find_root(key)
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/relation_tree_restorer.rb b/lib/gitlab/import_export/relation_tree_restorer.rb
index 8bc87ecb071..46b82240ef7 100644
--- a/lib/gitlab/import_export/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/relation_tree_restorer.rb
@@ -48,7 +48,7 @@ module Gitlab
@importable.reload # rubocop:disable Cop/ActiveRecordAssociationReload
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
@@ -81,7 +81,7 @@ module Gitlab
relation_object.save!
log_relation_creation(@importable, relation_key, relation_object)
end
- rescue => e
+ rescue StandardError => e
import_failure_service.log_import_failure(
source: 'process_relation_item!',
relation_key: relation_key,
@@ -155,7 +155,7 @@ module Gitlab
transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition, relation_index)
end
- relation = @relation_factory.create(**relation_factory_params(relation_key, data_hash))
+ relation = @relation_factory.create(**relation_factory_params(relation_key, relation_index, data_hash))
if relation && !relation.valid?
@shared.logger.warn(
@@ -221,8 +221,9 @@ module Gitlab
importable_class.to_s.downcase.to_sym
end
- def relation_factory_params(relation_key, data_hash)
+ def relation_factory_params(relation_key, relation_index, data_hash)
{
+ relation_index: relation_index,
relation_sym: relation_key.to_sym,
relation_hash: data_hash,
importable: @importable,
diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb
index 998da3e4afb..1c6629cf942 100644
--- a/lib/gitlab/import_export/repo_restorer.rb
+++ b/lib/gitlab/import_export/repo_restorer.rb
@@ -22,7 +22,7 @@ module Gitlab
update_importable_repository_info
true
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
end
@@ -52,4 +52,4 @@ module Gitlab
end
end
-Gitlab::ImportExport::RepoRestorer.prepend_if_ee('EE::Gitlab::ImportExport::RepoRestorer')
+Gitlab::ImportExport::RepoRestorer.prepend_mod_with('Gitlab::ImportExport::RepoRestorer')
diff --git a/lib/gitlab/import_export/repo_saver.rb b/lib/gitlab/import_export/repo_saver.rb
index 0fdd0722b65..fae07039139 100644
--- a/lib/gitlab/import_export/repo_saver.rb
+++ b/lib/gitlab/import_export/repo_saver.rb
@@ -40,7 +40,7 @@ module Gitlab
mkdir_p(File.dirname(bundle_full_path))
repository.bundle_to_disk(bundle_full_path)
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/saver.rb b/lib/gitlab/import_export/saver.rb
index bb2bbda4bd6..bec709f4a36 100644
--- a/lib/gitlab/import_export/saver.rb
+++ b/lib/gitlab/import_export/saver.rb
@@ -27,7 +27,7 @@ module Gitlab
@shared.error(Gitlab::ImportExport::Error.new(error_message))
false
end
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
ensure
diff --git a/lib/gitlab/import_export/shared.rb b/lib/gitlab/import_export/shared.rb
index 09ed4eb568d..f295ab38de0 100644
--- a/lib/gitlab/import_export/shared.rb
+++ b/lib/gitlab/import_export/shared.rb
@@ -90,7 +90,7 @@ module Gitlab
when 'Group'
@exportable.full_path
else
- raise Gitlab::ImportExport::Error.new("Unsupported Exportable Type #{@exportable&.class}")
+ raise Gitlab::ImportExport::Error, "Unsupported Exportable Type #{@exportable&.class}"
end
end
diff --git a/lib/gitlab/import_export/snippet_repo_restorer.rb b/lib/gitlab/import_export/snippet_repo_restorer.rb
index 2d0aa05fc3c..cb13972f8f2 100644
--- a/lib/gitlab/import_export/snippet_repo_restorer.rb
+++ b/lib/gitlab/import_export/snippet_repo_restorer.rb
@@ -23,7 +23,7 @@ module Gitlab
end
true
- rescue => e
+ rescue StandardError => e
shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/statistics_restorer.rb b/lib/gitlab/import_export/statistics_restorer.rb
index 3fafb01c37c..a3ad5edc2cc 100644
--- a/lib/gitlab/import_export/statistics_restorer.rb
+++ b/lib/gitlab/import_export/statistics_restorer.rb
@@ -10,7 +10,7 @@ module Gitlab
def restore
@project.statistics.refresh!
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/uploads_manager.rb b/lib/gitlab/import_export/uploads_manager.rb
index 2f15cdd7506..ad19508fb99 100644
--- a/lib/gitlab/import_export/uploads_manager.rb
+++ b/lib/gitlab/import_export/uploads_manager.rb
@@ -17,7 +17,7 @@ module Gitlab
copy_project_uploads
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
@@ -30,7 +30,7 @@ module Gitlab
end
true
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/uploads_restorer.rb b/lib/gitlab/import_export/uploads_restorer.rb
index 5f422dcbefa..741c6555aad 100644
--- a/lib/gitlab/import_export/uploads_restorer.rb
+++ b/lib/gitlab/import_export/uploads_restorer.rb
@@ -8,7 +8,7 @@ module Gitlab
project: @project,
shared: @shared
).restore
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/uploads_saver.rb b/lib/gitlab/import_export/uploads_saver.rb
index be1066c30b2..9f58609fa17 100644
--- a/lib/gitlab/import_export/uploads_saver.rb
+++ b/lib/gitlab/import_export/uploads_saver.rb
@@ -13,7 +13,7 @@ module Gitlab
project: @project,
shared: @shared
).save
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/version_checker.rb b/lib/gitlab/import_export/version_checker.rb
index 48f5b558e52..5ec9db00d0a 100644
--- a/lib/gitlab/import_export/version_checker.rb
+++ b/lib/gitlab/import_export/version_checker.rb
@@ -14,7 +14,7 @@ module Gitlab
def check!
version = File.open(version_file, &:readline)
verify_version!(version)
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
@@ -27,7 +27,7 @@ module Gitlab
def verify_version!(version)
if different_version?(version)
- raise Gitlab::ImportExport::Error.new("Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}")
+ raise Gitlab::ImportExport::Error, "Import version mismatch: Required #{Gitlab::ImportExport.version} but was #{version}"
else
true
end
@@ -35,13 +35,13 @@ module Gitlab
def different_version?(version)
Gem::Version.new(version) != Gem::Version.new(Gitlab::ImportExport.version)
- rescue => e
+ rescue StandardError => e
Gitlab::Import::Logger.error(
message: 'Import error',
error: e.message
)
- raise Gitlab::ImportExport::Error.new('Incorrect VERSION format')
+ raise Gitlab::ImportExport::Error, 'Incorrect VERSION format'
end
end
end
diff --git a/lib/gitlab/import_export/version_saver.rb b/lib/gitlab/import_export/version_saver.rb
index dab8bbf539d..e8f68f93af0 100644
--- a/lib/gitlab/import_export/version_saver.rb
+++ b/lib/gitlab/import_export/version_saver.rb
@@ -15,7 +15,7 @@ module Gitlab
File.write(version_file, Gitlab::ImportExport.version, mode: 'w')
File.write(gitlab_version_file, Gitlab::VERSION, mode: 'w')
File.write(gitlab_revision_file, Gitlab.revision, mode: 'w')
- rescue => e
+ rescue StandardError => e
@shared.error(e)
false
end
diff --git a/lib/gitlab/import_export/wiki_repo_saver.rb b/lib/gitlab/import_export/wiki_repo_saver.rb
index 4b1cf4915e4..162b474bfeb 100644
--- a/lib/gitlab/import_export/wiki_repo_saver.rb
+++ b/lib/gitlab/import_export/wiki_repo_saver.rb
@@ -20,4 +20,4 @@ module Gitlab
end
end
-Gitlab::ImportExport::WikiRepoSaver.prepend_if_ee('EE::Gitlab::ImportExport::WikiRepoSaver')
+Gitlab::ImportExport::WikiRepoSaver.prepend_mod_with('Gitlab::ImportExport::WikiRepoSaver')