diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 12:40:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-10-20 12:40:42 +0300 |
commit | ee664acb356f8123f4f6b00b73c1e1cf0866c7fb (patch) | |
tree | f8479f94a28f66654c6a4f6fb99bad6b4e86a40e /lib/bulk_imports | |
parent | 62f7d5c5b69180e82ae8196b7b429eeffc8e7b4f (diff) |
Add latest changes from gitlab-org/gitlab@15-5-stable-eev15.5.0-rc42
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r-- | lib/bulk_imports/common/pipelines/entity_finisher.rb | 5 | ||||
-rw-r--r-- | lib/bulk_imports/common/pipelines/wiki_pipeline.rb | 5 | ||||
-rw-r--r-- | lib/bulk_imports/common/rest/get_badges_query.rb | 5 | ||||
-rw-r--r-- | lib/bulk_imports/features.rb | 19 | ||||
-rw-r--r-- | lib/bulk_imports/groups/extractors/subgroups_extractor.rb | 4 | ||||
-rw-r--r-- | lib/bulk_imports/groups/graphql/get_group_query.rb | 2 | ||||
-rw-r--r-- | lib/bulk_imports/groups/graphql/get_projects_query.rb | 1 | ||||
-rw-r--r-- | lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb | 3 | ||||
-rw-r--r-- | lib/bulk_imports/groups/stage.rb | 8 | ||||
-rw-r--r-- | lib/bulk_imports/groups/transformers/group_attributes_transformer.rb | 97 | ||||
-rw-r--r-- | lib/bulk_imports/network_error.rb | 34 | ||||
-rw-r--r-- | lib/bulk_imports/pipeline.rb | 6 | ||||
-rw-r--r-- | lib/bulk_imports/pipeline/context.rb | 4 | ||||
-rw-r--r-- | lib/bulk_imports/pipeline/runner.rb | 13 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/get_project_query.rb | 1 |
15 files changed, 122 insertions, 85 deletions
diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 915dcf1b455..5066f622d57 100644 --- a/lib/bulk_imports/common/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -24,11 +24,12 @@ module BulkImports end logger.info( - bulk_import_id: context.bulk_import.id, + bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, pipeline_class: self.class.name, - message: "Entity #{entity.status_name}" + message: "Entity #{entity.status_name}", + importer: 'gitlab_migration' ) context.portable.try(:after_import) diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index 0f92c1f1210..fea550b9f9d 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -44,6 +44,11 @@ module BulkImports wikis = client.get(context.entity.wikis_url_path).parsed_response wikis.any? + rescue BulkImports::NetworkError => e + # 403 is returned when wiki is disabled in settings + return if e.response&.forbidden? || e.response&.not_found? + + raise end def client diff --git a/lib/bulk_imports/common/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb index 60b2ebcc552..6e7cb7504ba 100644 --- a/lib/bulk_imports/common/rest/get_badges_query.rb +++ b/lib/bulk_imports/common/rest/get_badges_query.rb @@ -7,11 +7,8 @@ module BulkImports extend self def to_h(context) - resource = context.entity.pluralized_name - encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path) - { - resource: [resource, encoded_full_path, 'badges'].join('/'), + resource: [context.entity.base_resource_path, 'badges'].join('/'), query: { page: context.tracker.next_page } diff --git a/lib/bulk_imports/features.rb b/lib/bulk_imports/features.rb new file mode 100644 index 00000000000..952e8e62d71 --- /dev/null +++ b/lib/bulk_imports/features.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Features + def self.enabled? + ::Feature.enabled?(:bulk_import) + end + + def self.project_migration_enabled?(destination_namespace = nil) + if destination_namespace.present? + root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor + + ::Feature.enabled?(:bulk_import_projects, root_ancestor) + else + ::Feature.enabled?(:bulk_import_projects) + end + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb index 1140beef48c..029b6f0f729 100644 --- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -5,10 +5,8 @@ module BulkImports module Extractors class SubgroupsExtractor def extract(context) - encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) - response = http_client(context.configuration) - .each_page(:get, "groups/#{encoded_parent_path}/subgroups") + .each_page(:get, "#{context.entity.base_resource_path}/subgroups") .flat_map(&:itself) BulkImports::Pipeline::ExtractedData.new(data: response) diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb index 911b2b67d8c..0e73a7fb4b9 100644 --- a/lib/bulk_imports/groups/graphql/get_group_query.rb +++ b/lib/bulk_imports/groups/graphql/get_group_query.rb @@ -14,9 +14,9 @@ module BulkImports <<-'GRAPHQL' query($full_path: ID!) { group(fullPath: $full_path) { + id name path - full_path: fullPath description visibility emails_disabled: emailsDisabled diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb index 3f74bbb8cce..4784baf225c 100644 --- a/lib/bulk_imports/groups/graphql/get_projects_query.rb +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -20,6 +20,7 @@ module BulkImports has_next_page: hasNextPage } nodes { + id name full_path: fullPath } diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb index c318675e649..026b2e55713 100644 --- a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -15,7 +15,8 @@ module BulkImports source_full_path: data['full_path'], destination_name: data['name'], destination_namespace: context.entity.group.full_path, - parent_id: context.entity.id + parent_id: context.entity.id, + source_xid: GlobalID.parse(data['id']).model_id } end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index 0378a9c605d..6928ce43191 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -90,13 +90,7 @@ module BulkImports def feature_flag_enabled? destination_namespace = @bulk_import_entity.destination_namespace - if destination_namespace.present? - root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor - - ::Feature.enabled?(:bulk_import_projects, root_ancestor) - else - ::Feature.enabled?(:bulk_import_projects) - end + BulkImports::Features.project_migration_enabled?(destination_namespace) end end end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index d9efcdb1ba5..83b442458dc 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -4,6 +4,7 @@ module BulkImports module Groups module Transformers class GroupAttributesTransformer + # rubocop: disable Style/IfUnlessModifier def transform(context, data) import_entity = context.entity @@ -11,74 +12,64 @@ module BulkImports namespace = Namespace.find_by_full_path(import_entity.destination_namespace) end - data - .then { |data| transform_name(import_entity, namespace, data) } - .then { |data| transform_path(import_entity, data) } - .then { |data| transform_full_path(data) } - .then { |data| transform_parent(context, import_entity, namespace, data) } - .then { |data| transform_visibility_level(data) } - .then { |data| transform_project_creation_level(data) } - .then { |data| transform_subgroup_creation_level(data) } - end - - private + params = { + 'name' => group_name(namespace, data), + 'path' => import_entity.destination_slug.parameterize, + 'description' => data['description'], + 'lfs_enabled' => data['lfs_enabled'], + 'emails_disabled' => data['emails_disabled'], + 'mentions_disabled' => data['mentions_disabled'], + 'share_with_group_lock' => data['share_with_group_lock'] + } - def transform_name(import_entity, namespace, data) if namespace.present? - namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord - - if namespace_children_names.include?(data['name']) - data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base| - namespace_children_names.include?(base) - end - end + params['parent_id'] = namespace.id end - data - end - - def transform_path(import_entity, data) - data['path'] = import_entity.destination_slug.parameterize - data - end + if data.has_key?('two_factor_grace_period') + params['two_factor_grace_period'] = data['two_factor_grace_period'] + end - def transform_full_path(data) - data.delete('full_path') - data - end + if data.has_key?('request_access_enabled') + params['request_access_enabled'] = data['request_access_enabled'] + end - def transform_parent(context, import_entity, namespace, data) - data['parent_id'] = namespace.id if namespace.present? + if data.has_key?('require_two_factor_authentication') + params['require_two_factor_authentication'] = data['require_two_factor_authentication'] + end - data - end + if data.has_key?('project_creation_level') + params['project_creation_level'] = + Gitlab::Access.project_creation_string_options[data['project_creation_level']] + end - def transform_visibility_level(data) - visibility = data['visibility'] + if data.has_key?('subgroup_creation_level') + params['subgroup_creation_level'] = + Gitlab::Access.subgroup_creation_string_options[data['subgroup_creation_level']] + end - return data unless visibility.present? + if data.has_key?('visibility') + params['visibility_level'] = Gitlab::VisibilityLevel.string_options[data['visibility']] + end - data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] - data.delete('visibility') - data + params end + # rubocop: enable Style/IfUnlessModifier - def transform_project_creation_level(data) - project_creation_level = data['project_creation_level'] - - return data unless project_creation_level.present? - - data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level] - data - end + private - def transform_subgroup_creation_level(data) - subgroup_creation_level = data['subgroup_creation_level'] + def group_name(namespace, data) + if namespace.present? + namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord - return data unless subgroup_creation_level.present? + if namespace_children_names.include?(data['name']) + data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base| + namespace_children_names.include?(base) + end + end + end - data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level] - data + data['name'] end end end diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb index 3514291a75d..fda4bb74a30 100644 --- a/lib/bulk_imports/network_error.rb +++ b/lib/bulk_imports/network_error.rb @@ -2,15 +2,21 @@ module BulkImports class NetworkError < Error - COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}' + ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}' - RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [ + EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError, + Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH + ].freeze RETRIABLE_HTTP_CODES = [429].freeze DEFAULT_RETRY_DELAY_SECONDS = 30 MAX_RETRIABLE_COUNT = 10 + attr_reader :response + def initialize(message = nil, response: nil) raise ArgumentError, 'message or response required' if message.blank? && response.blank? @@ -19,9 +25,9 @@ module BulkImports @response = response end - def retriable?(tracker) + def retriable?(object) if retriable_exception? || retriable_http_code? - increment(tracker) <= MAX_RETRIABLE_COUNT + increment(object) <= MAX_RETRIABLE_COUNT else false end @@ -37,8 +43,6 @@ module BulkImports private - attr_reader :response - def retriable_exception? RETRIABLE_EXCEPTIONS.include?(cause&.class) end @@ -47,15 +51,27 @@ module BulkImports RETRIABLE_HTTP_CODES.include?(response&.code) end - def increment(tracker) - key = COUNTER_KEY % { + def increment(object) + key = object.is_a?(BulkImports::Entity) ? entity_cache_key(object) : tracker_cache_key(object) + + Gitlab::Cache::Import::Caching.increment(key) + end + + def tracker_cache_key(tracker) + TRACKER_COUNTER_KEY % { stage: tracker.stage, tracker_id: tracker.id, entity_id: tracker.entity.id, error: cause.class.name } + end - Gitlab::Cache::Import::Caching.increment(key) + def entity_cache_key(entity) + ENTITY_COUNTER_KEY % { + import_id: entity.bulk_import_id, + entity_id: entity.id, + error: cause.class.name + } end end end diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index 68b86c68619..54d5d3209f8 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -183,7 +183,11 @@ module BulkImports end def relation - class_attributes[:relation_name] + class_attributes[:relation_name] || default_relation + end + + def default_relation + self.name.demodulize.chomp('Pipeline').underscore end private diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index d753f888671..fde24cf3646 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -32,6 +32,10 @@ module BulkImports @bulk_import ||= entity.bulk_import end + def bulk_import_id + @bulk_import_id ||= bulk_import.id + end + def current_user @current_user ||= bulk_import.user end diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb index c03da7d8d01..ef9575d1e96 100644 --- a/lib/bulk_imports/pipeline/runner.rb +++ b/lib/bulk_imports/pipeline/runner.rb @@ -54,7 +54,8 @@ module BulkImports skip!( 'Skipping pipeline due to failed entity', pipeline_step: step, - step_class: class_name + step_class: class_name, + importer: 'gitlab_migration' ) rescue BulkImports::NetworkError => e if e.retriable?(context.tracker) @@ -108,9 +109,12 @@ module BulkImports } error( + bulk_import_id: context.bulk_import_id, pipeline_step: step, exception_class: exception.class.to_s, - exception_message: exception.message + exception_message: exception.message, + message: "Pipeline failed", + importer: 'gitlab_migration' ) BulkImports::Failure.create(attributes) @@ -130,11 +134,12 @@ module BulkImports def log_params(extra) defaults = { - bulk_import_id: context.bulk_import.id, + bulk_import_id: context.bulk_import_id, bulk_import_entity_id: context.entity.id, bulk_import_entity_type: context.entity.source_type, pipeline_class: pipeline, - context_extra: context.extra + context_extra: context.extra, + importer: 'gitlab_migration' } defaults diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 76475893ac1..a2d7094d570 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -10,6 +10,7 @@ module BulkImports <<-'GRAPHQL' query($full_path: ID!) { project(fullPath: $full_path) { + id visibility created_at: createdAt } |