diff options
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r-- | lib/bulk_imports/common/graphql/get_members_query.rb | 23 | ||||
-rw-r--r-- | lib/bulk_imports/common/pipelines/entity_finisher.rb | 2 | ||||
-rw-r--r-- | lib/bulk_imports/common/transformers/member_attributes_transformer.rb | 13 | ||||
-rw-r--r-- | lib/bulk_imports/groups/loaders/group_loader.rb | 12 | ||||
-rw-r--r-- | lib/bulk_imports/network_error.rb | 13 | ||||
-rw-r--r-- | lib/bulk_imports/pipeline/runner.rb | 8 | ||||
-rw-r--r-- | lib/bulk_imports/projects/pipelines/references_pipeline.rb | 25 | ||||
-rw-r--r-- | lib/bulk_imports/users_mapper.rb | 41 |
8 files changed, 111 insertions, 26 deletions
diff --git a/lib/bulk_imports/common/graphql/get_members_query.rb b/lib/bulk_imports/common/graphql/get_members_query.rb index 8fa8d7f4c0b..4f7533ee25f 100644 --- a/lib/bulk_imports/common/graphql/get_members_query.rb +++ b/lib/bulk_imports/common/graphql/get_members_query.rb @@ -14,7 +14,7 @@ module BulkImports <<-GRAPHQL query($full_path: ID!, $cursor: String, $per_page: Int) { portable: #{context.entity.entity_type}(fullPath: $full_path) { - members: #{members_type}(relations: #{relations}, first: $per_page, after: $cursor) { + members: #{members_type}(relations: [#{relations}], first: $per_page, after: $cursor) { page_info: pageInfo { next_page: endCursor has_next_page: hasNextPage @@ -29,6 +29,7 @@ module BulkImports user { user_gid: id public_email: publicEmail + username: username } } } @@ -69,11 +70,27 @@ module BulkImports def relations if context.entity.group? - "[DIRECT INHERITED SHARED_FROM_GROUPS]" + group_relations else - "[DIRECT INHERITED INVITED_GROUPS SHARED_INTO_ANCESTORS]" + project_relations end end + + def source_version + Gitlab::VersionInfo.parse(context.bulk_import.source_version) + end + + def group_relations + base_relation = "DIRECT INHERITED" + base_relation += " SHARED_FROM_GROUPS" if source_version >= Gitlab::VersionInfo.parse("14.7.0") + base_relation + end + + def project_relations + base_relation = "DIRECT INHERITED INVITED_GROUPS" + base_relation += " SHARED_INTO_ANCESTORS" if source_version >= Gitlab::VersionInfo.parse("16.0.0") + base_relation + end end end end diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index a52504d04bc..fa09f36fdd6 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -34,7 +34,7 @@ module BulkImports importer: 'gitlab_migration' ) - context.portable.try(:after_import) + ::BulkImports::FinishProjectImportWorker.perform_async(entity.project_id) if entity.project? end private diff --git a/lib/bulk_imports/common/transformers/member_attributes_transformer.rb b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb index 382e6a51a73..4cd87ec2b59 100644 --- a/lib/bulk_imports/common/transformers/member_attributes_transformer.rb +++ b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb @@ -12,7 +12,7 @@ module BulkImports return unless user return unless valid_access_level?(access_level) - cache_source_user_id(data, user, context) + cache_source_user_data(data, user, context) { user_id: user.id, @@ -36,14 +36,21 @@ module BulkImports Gitlab::Access.options_with_owner.value?(access_level) end - def cache_source_user_id(data, user, context) + def cache_source_user_data(data, user, context) gid = data&.dig('user', 'user_gid') return unless gid source_user_id = GlobalID.parse(gid).model_id + source_username = data&.dig('user', 'username') - ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id) + mapper = ::BulkImports::UsersMapper.new(context: context) + + mapper.cache_source_user_id(source_user_id, user.id) + return unless source_username + return if source_username == user.username + + mapper.cache_source_username(source_username, user.username) end end end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb index 5f5307123a5..85d85f0f703 100644 --- a/lib/bulk_imports/groups/loaders/group_loader.rb +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -4,6 +4,8 @@ module BulkImports module Groups module Loaders class GroupLoader + TWO_FACTOR_KEY = 'require_two_factor_authentication' + GroupCreationError = Class.new(StandardError) def load(context, data) @@ -16,6 +18,10 @@ module BulkImports raise(GroupCreationError, 'User not allowed to create group') unless user_can_create_group?(current_user, data) raise(GroupCreationError, 'Group exists') if group_exists?(destination_namespace, path) + unless two_factor_requirements_met?(current_user, data) + raise(GroupCreationError, 'User requires Two-Factor Authentication') + end + group = ::Groups::CreateService.new(current_user, data).execute raise(GroupCreationError, group.errors.full_messages.to_sentence) if group.errors.any? @@ -37,6 +43,12 @@ module BulkImports end end + def two_factor_requirements_met?(current_user, data) + return true unless data.has_key?(TWO_FACTOR_KEY) && data[TWO_FACTOR_KEY] + + current_user.two_factor_enabled? + end + def group_exists?(destination_namespace, path) full_path = destination_namespace.present? ? File.join(destination_namespace, path) : path diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index fda4bb74a30..b21889adcb3 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -5,10 +5,19 @@ module BulkImports TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}' - RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [ + NO_SPACE_LEFT_EXCEPTION = Errno::ENOSPC + DECOMPRESSION_FAILURE_EXCEPTION = Zlib::Error + + EXCEPTIONS_RETRY_DELAY = { + NO_SPACE_LEFT_EXCEPTION => 120, + DECOMPRESSION_FAILURE_EXCEPTION => 60 + }.freeze + + RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + EXCEPTIONS_RETRY_DELAY.keys + [ EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH ].freeze + RETRIABLE_HTTP_CODES = [429].freeze DEFAULT_RETRY_DELAY_SECONDS = 30 @@ -37,7 +46,7 @@ module BulkImports if response&.code == 429 response.headers.fetch('Retry-After', DEFAULT_RETRY_DELAY_SECONDS).to_i else - DEFAULT_RETRY_DELAY_SECONDS + EXCEPTIONS_RETRY_DELAY[cause&.class] || DEFAULT_RETRY_DELAY_SECONDS end.seconds end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index 81f8dee30d9..1e2d9152047 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -58,11 +58,9 @@ module BulkImports importer: 'gitlab_migration' ) rescue BulkImports::NetworkError => e - if e.retriable?(context.tracker) - raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay) - else - log_and_fail(e, step) - end + raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay) if e.retriable?(context.tracker) + + log_and_fail(e, step) rescue BulkImports::RetryPipelineError raise rescue StandardError => e diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb index f00a62edf51..e2032569ab5 100644 --- a/lib/bulk_imports/projects/pipelines/references_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb @@ -22,8 +22,10 @@ module BulkImports def transform(_context, object) body = object_body(object).dup + body.gsub!(username_regex(mapped_usernames), mapped_usernames) + matching_urls(object).each do |old_url, new_url| - body.gsub!(old_url, new_url) + body.gsub!(old_url, new_url) if body.include?(old_url) end object.assign_attributes(body_field(object) => body) @@ -37,10 +39,21 @@ module BulkImports private + def mapped_usernames + @mapped_usernames ||= ::BulkImports::UsersMapper.new(context: context) + .map_usernames.transform_keys { |key| "@#{key}" } + .transform_values { |value| "@#{value}" } + end + + def username_regex(mapped_usernames) + @username_regex ||= Regexp.new(mapped_usernames.keys.sort_by(&:length) + .reverse.map { |x| Regexp.escape(x) }.join('|')) + end + def add_matching_objects(collection, enum) collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| batch.each do |object| - enum << object if object_has_reference?(object) + enum << object if object_has_reference?(object) || object_has_username?(object) end end end @@ -51,7 +64,7 @@ module BulkImports object.notes.each_batch(of: BATCH_SIZE) do |notes_batch| notes_batch.each do |note| note.refresh_markdown_cache! - enum << note if object_has_reference?(note) + enum << note if object_has_reference?(note) || object_has_username?(note) end end end @@ -62,6 +75,12 @@ module BulkImports object_body(object)&.include?(source_full_path) end + def object_has_username?(object) + return false unless object_body(object) + + mapped_usernames.keys.any? { |old_username| object_body(object).include?(old_username) } + end + def object_body(object) call_object_method(object) end diff --git a/lib/bulk_imports/users_mapper.rb b/lib/bulk_imports/users_mapper.rb index 74412bc3831..af002c67367 100644 --- a/lib/bulk_imports/users_mapper.rb +++ b/lib/bulk_imports/users_mapper.rb @@ -6,17 +6,17 @@ module BulkImports SOURCE_USER_IDS_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_user_ids' + SOURCE_USERNAMES_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_usernames' + def initialize(context:) @context = context - @cache_key = SOURCE_USER_IDS_CACHE_KEY % { - bulk_import: @context.bulk_import.id, - entity: @context.entity.id - } + @user_ids_cache_key = generate_cache_key(SOURCE_USER_IDS_CACHE_KEY) + @usernames_cache_key = generate_cache_key(SOURCE_USERNAMES_CACHE_KEY) end def map strong_memoize(:map) do - map = hash_with_default + map = Hash.new { default_user_id } cached_source_user_ids.each_pair do |source_id, destination_id| map[source_id.to_i] = destination_id.to_i @@ -26,6 +26,18 @@ module BulkImports end end + def map_usernames + strong_memoize(:map_usernames) do + map = {} + + cached_source_usernames.each_pair do |source_username, destination_username| + map[source_username] = destination_username + end + + map + end + end + def include?(source_user_id) map.has_key?(source_user_id) end @@ -35,17 +47,28 @@ module BulkImports end def cache_source_user_id(source_id, destination_id) - ::Gitlab::Cache::Import::Caching.hash_add(@cache_key, source_id, destination_id) + ::Gitlab::Cache::Import::Caching.hash_add(@user_ids_cache_key, source_id, destination_id) + end + + def cache_source_username(source_username, destination_username) + ::Gitlab::Cache::Import::Caching.hash_add(@usernames_cache_key, source_username, destination_username) end private - def hash_with_default - Hash.new { default_user_id } + def generate_cache_key(pattern) + pattern % { + bulk_import: @context.bulk_import.id, + entity: @context.entity.id + } end def cached_source_user_ids - ::Gitlab::Cache::Import::Caching.values_from_hash(@cache_key) + ::Gitlab::Cache::Import::Caching.values_from_hash(@user_ids_cache_key) + end + + def cached_source_usernames + ::Gitlab::Cache::Import::Caching.values_from_hash(@usernames_cache_key) end end end |