Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
path: root/lib/bulk_imports
diff options
context:
space:
mode:
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r--lib/bulk_imports/common/graphql/get_members_query.rb23
-rw-r--r--lib/bulk_imports/common/pipelines/entity_finisher.rb2
-rw-r--r--lib/bulk_imports/common/transformers/member_attributes_transformer.rb13
-rw-r--r--lib/bulk_imports/groups/loaders/group_loader.rb12
-rw-r--r--lib/bulk_imports/network_error.rb13
-rw-r--r--lib/bulk_imports/pipeline/runner.rb8
-rw-r--r--lib/bulk_imports/projects/pipelines/references_pipeline.rb25
-rw-r--r--lib/bulk_imports/users_mapper.rb41
8 files changed, 111 insertions, 26 deletions
diff --git a/lib/bulk_imports/common/graphql/get_members_query.rb b/lib/bulk_imports/common/graphql/get_members_query.rb
index 8fa8d7f4c0b..4f7533ee25f 100644
--- a/lib/bulk_imports/common/graphql/get_members_query.rb
+++ b/lib/bulk_imports/common/graphql/get_members_query.rb
@@ -14,7 +14,7 @@ module BulkImports
<<-GRAPHQL
query($full_path: ID!, $cursor: String, $per_page: Int) {
portable: #{context.entity.entity_type}(fullPath: $full_path) {
- members: #{members_type}(relations: #{relations}, first: $per_page, after: $cursor) {
+ members: #{members_type}(relations: [#{relations}], first: $per_page, after: $cursor) {
page_info: pageInfo {
next_page: endCursor
has_next_page: hasNextPage
@@ -29,6 +29,7 @@ module BulkImports
user {
user_gid: id
public_email: publicEmail
+ username: username
}
}
}
@@ -69,11 +70,27 @@ module BulkImports
def relations
if context.entity.group?
- "[DIRECT INHERITED SHARED_FROM_GROUPS]"
+ group_relations
else
- "[DIRECT INHERITED INVITED_GROUPS SHARED_INTO_ANCESTORS]"
+ project_relations
end
end
+
+ def source_version
+ Gitlab::VersionInfo.parse(context.bulk_import.source_version)
+ end
+
+ def group_relations
+ base_relation = "DIRECT INHERITED"
+ base_relation += " SHARED_FROM_GROUPS" if source_version >= Gitlab::VersionInfo.parse("14.7.0")
+ base_relation
+ end
+
+ def project_relations
+ base_relation = "DIRECT INHERITED INVITED_GROUPS"
+ base_relation += " SHARED_INTO_ANCESTORS" if source_version >= Gitlab::VersionInfo.parse("16.0.0")
+ base_relation
+ end
end
end
end
diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb
index a52504d04bc..fa09f36fdd6 100644
--- a/lib/bulk_imports/common/pipelines/entity_finisher.rb
+++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb
@@ -34,7 +34,7 @@ module BulkImports
importer: 'gitlab_migration'
)
- context.portable.try(:after_import)
+ ::BulkImports::FinishProjectImportWorker.perform_async(entity.project_id) if entity.project?
end
private
diff --git a/lib/bulk_imports/common/transformers/member_attributes_transformer.rb b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb
index 382e6a51a73..4cd87ec2b59 100644
--- a/lib/bulk_imports/common/transformers/member_attributes_transformer.rb
+++ b/lib/bulk_imports/common/transformers/member_attributes_transformer.rb
@@ -12,7 +12,7 @@ module BulkImports
return unless user
return unless valid_access_level?(access_level)
- cache_source_user_id(data, user, context)
+ cache_source_user_data(data, user, context)
{
user_id: user.id,
@@ -36,14 +36,21 @@ module BulkImports
Gitlab::Access.options_with_owner.value?(access_level)
end
- def cache_source_user_id(data, user, context)
+ def cache_source_user_data(data, user, context)
gid = data&.dig('user', 'user_gid')
return unless gid
source_user_id = GlobalID.parse(gid).model_id
+ source_username = data&.dig('user', 'username')
- ::BulkImports::UsersMapper.new(context: context).cache_source_user_id(source_user_id, user.id)
+ mapper = ::BulkImports::UsersMapper.new(context: context)
+
+ mapper.cache_source_user_id(source_user_id, user.id)
+ return unless source_username
+ return if source_username == user.username
+
+ mapper.cache_source_username(source_username, user.username)
end
end
end
diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb
index 5f5307123a5..85d85f0f703 100644
--- a/lib/bulk_imports/groups/loaders/group_loader.rb
+++ b/lib/bulk_imports/groups/loaders/group_loader.rb
@@ -4,6 +4,8 @@ module BulkImports
module Groups
module Loaders
class GroupLoader
+ TWO_FACTOR_KEY = 'require_two_factor_authentication'
+
GroupCreationError = Class.new(StandardError)
def load(context, data)
@@ -16,6 +18,10 @@ module BulkImports
raise(GroupCreationError, 'User not allowed to create group') unless user_can_create_group?(current_user, data)
raise(GroupCreationError, 'Group exists') if group_exists?(destination_namespace, path)
+ unless two_factor_requirements_met?(current_user, data)
+ raise(GroupCreationError, 'User requires Two-Factor Authentication')
+ end
+
group = ::Groups::CreateService.new(current_user, data).execute
raise(GroupCreationError, group.errors.full_messages.to_sentence) if group.errors.any?
@@ -37,6 +43,12 @@ module BulkImports
end
end
+ def two_factor_requirements_met?(current_user, data)
+ return true unless data.has_key?(TWO_FACTOR_KEY) && data[TWO_FACTOR_KEY]
+
+ current_user.two_factor_enabled?
+ end
+
def group_exists?(destination_namespace, path)
full_path = destination_namespace.present? ? File.join(destination_namespace, path) : path
diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb
index fda4bb74a30..b21889adcb3 100644
--- a/lib/bulk_imports/network_error.rb
+++ b/lib/bulk_imports/network_error.rb
@@ -5,10 +5,19 @@ module BulkImports
TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}'
ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}'
- RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [
+ NO_SPACE_LEFT_EXCEPTION = Errno::ENOSPC
+ DECOMPRESSION_FAILURE_EXCEPTION = Zlib::Error
+
+ EXCEPTIONS_RETRY_DELAY = {
+ NO_SPACE_LEFT_EXCEPTION => 120,
+ DECOMPRESSION_FAILURE_EXCEPTION => 60
+ }.freeze
+
+ RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + EXCEPTIONS_RETRY_DELAY.keys + [
EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH
].freeze
+
RETRIABLE_HTTP_CODES = [429].freeze
DEFAULT_RETRY_DELAY_SECONDS = 30
@@ -37,7 +46,7 @@ module BulkImports
if response&.code == 429
response.headers.fetch('Retry-After', DEFAULT_RETRY_DELAY_SECONDS).to_i
else
- DEFAULT_RETRY_DELAY_SECONDS
+ EXCEPTIONS_RETRY_DELAY[cause&.class] || DEFAULT_RETRY_DELAY_SECONDS
end.seconds
end
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index 81f8dee30d9..1e2d9152047 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -58,11 +58,9 @@ module BulkImports
importer: 'gitlab_migration'
)
rescue BulkImports::NetworkError => e
- if e.retriable?(context.tracker)
- raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay)
- else
- log_and_fail(e, step)
- end
+ raise BulkImports::RetryPipelineError.new(e.message, e.retry_delay) if e.retriable?(context.tracker)
+
+ log_and_fail(e, step)
rescue BulkImports::RetryPipelineError
raise
rescue StandardError => e
diff --git a/lib/bulk_imports/projects/pipelines/references_pipeline.rb b/lib/bulk_imports/projects/pipelines/references_pipeline.rb
index f00a62edf51..e2032569ab5 100644
--- a/lib/bulk_imports/projects/pipelines/references_pipeline.rb
+++ b/lib/bulk_imports/projects/pipelines/references_pipeline.rb
@@ -22,8 +22,10 @@ module BulkImports
def transform(_context, object)
body = object_body(object).dup
+ body.gsub!(username_regex(mapped_usernames), mapped_usernames)
+
matching_urls(object).each do |old_url, new_url|
- body.gsub!(old_url, new_url)
+ body.gsub!(old_url, new_url) if body.include?(old_url)
end
object.assign_attributes(body_field(object) => body)
@@ -37,10 +39,21 @@ module BulkImports
private
+ def mapped_usernames
+ @mapped_usernames ||= ::BulkImports::UsersMapper.new(context: context)
+ .map_usernames.transform_keys { |key| "@#{key}" }
+ .transform_values { |value| "@#{value}" }
+ end
+
+ def username_regex(mapped_usernames)
+ @username_regex ||= Regexp.new(mapped_usernames.keys.sort_by(&:length)
+ .reverse.map { |x| Regexp.escape(x) }.join('|'))
+ end
+
def add_matching_objects(collection, enum)
collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch|
batch.each do |object|
- enum << object if object_has_reference?(object)
+ enum << object if object_has_reference?(object) || object_has_username?(object)
end
end
end
@@ -51,7 +64,7 @@ module BulkImports
object.notes.each_batch(of: BATCH_SIZE) do |notes_batch|
notes_batch.each do |note|
note.refresh_markdown_cache!
- enum << note if object_has_reference?(note)
+ enum << note if object_has_reference?(note) || object_has_username?(note)
end
end
end
@@ -62,6 +75,12 @@ module BulkImports
object_body(object)&.include?(source_full_path)
end
+ def object_has_username?(object)
+ return false unless object_body(object)
+
+ mapped_usernames.keys.any? { |old_username| object_body(object).include?(old_username) }
+ end
+
def object_body(object)
call_object_method(object)
end
diff --git a/lib/bulk_imports/users_mapper.rb b/lib/bulk_imports/users_mapper.rb
index 74412bc3831..af002c67367 100644
--- a/lib/bulk_imports/users_mapper.rb
+++ b/lib/bulk_imports/users_mapper.rb
@@ -6,17 +6,17 @@ module BulkImports
SOURCE_USER_IDS_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_user_ids'
+ SOURCE_USERNAMES_CACHE_KEY = 'bulk_imports/%{bulk_import}/%{entity}/source_usernames'
+
def initialize(context:)
@context = context
- @cache_key = SOURCE_USER_IDS_CACHE_KEY % {
- bulk_import: @context.bulk_import.id,
- entity: @context.entity.id
- }
+ @user_ids_cache_key = generate_cache_key(SOURCE_USER_IDS_CACHE_KEY)
+ @usernames_cache_key = generate_cache_key(SOURCE_USERNAMES_CACHE_KEY)
end
def map
strong_memoize(:map) do
- map = hash_with_default
+ map = Hash.new { default_user_id }
cached_source_user_ids.each_pair do |source_id, destination_id|
map[source_id.to_i] = destination_id.to_i
@@ -26,6 +26,18 @@ module BulkImports
end
end
+ def map_usernames
+ strong_memoize(:map_usernames) do
+ map = {}
+
+ cached_source_usernames.each_pair do |source_username, destination_username|
+ map[source_username] = destination_username
+ end
+
+ map
+ end
+ end
+
def include?(source_user_id)
map.has_key?(source_user_id)
end
@@ -35,17 +47,28 @@ module BulkImports
end
def cache_source_user_id(source_id, destination_id)
- ::Gitlab::Cache::Import::Caching.hash_add(@cache_key, source_id, destination_id)
+ ::Gitlab::Cache::Import::Caching.hash_add(@user_ids_cache_key, source_id, destination_id)
+ end
+
+ def cache_source_username(source_username, destination_username)
+ ::Gitlab::Cache::Import::Caching.hash_add(@usernames_cache_key, source_username, destination_username)
end
private
- def hash_with_default
- Hash.new { default_user_id }
+ def generate_cache_key(pattern)
+ pattern % {
+ bulk_import: @context.bulk_import.id,
+ entity: @context.entity.id
+ }
end
def cached_source_user_ids
- ::Gitlab::Cache::Import::Caching.values_from_hash(@cache_key)
+ ::Gitlab::Cache::Import::Caching.values_from_hash(@user_ids_cache_key)
+ end
+
+ def cached_source_usernames
+ ::Gitlab::Cache::Import::Caching.values_from_hash(@usernames_cache_key)
end
end
end
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-18 13:45:19 +0300