Add latest changes from gitlab-org/gitlab@15-5-stable-eev15.5.0-rc42

15 files changed, 122 insertions, 85 deletions

diff --git a/lib/bulk_imports/common/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb
index 915dcf1b455..5066f622d57 100644
--- a/lib/bulk_imports/common/pipelines/entity_finisher.rb
+++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb
@@ -24,11 +24,12 @@ module BulkImports
           end
 
           logger.info(
-            bulk_import_id: context.bulk_import.id,
+            bulk_import_id: context.bulk_import_id,
             bulk_import_entity_id: context.entity.id,
             bulk_import_entity_type: context.entity.source_type,
             pipeline_class: self.class.name,
-            message: "Entity #{entity.status_name}"
+            message: "Entity #{entity.status_name}",
+            importer: 'gitlab_migration'
           )
 
           context.portable.try(:after_import)
diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb
index 0f92c1f1210..fea550b9f9d 100644
--- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb
+++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb
@@ -44,6 +44,11 @@ module BulkImports
           wikis = client.get(context.entity.wikis_url_path).parsed_response
 
           wikis.any?
+        rescue BulkImports::NetworkError => e
+          # 403 is returned when wiki is disabled in settings
+          return if e.response&.forbidden? || e.response&.not_found?
+
+          raise
         end
 
         def client
diff --git a/lib/bulk_imports/common/rest/get_badges_query.rb b/lib/bulk_imports/common/rest/get_badges_query.rb
index 60b2ebcc552..6e7cb7504ba 100644
--- a/lib/bulk_imports/common/rest/get_badges_query.rb
+++ b/lib/bulk_imports/common/rest/get_badges_query.rb
@@ -7,11 +7,8 @@ module BulkImports
         extend self
 
         def to_h(context)
-          resource = context.entity.pluralized_name
-          encoded_full_path = ERB::Util.url_encode(context.entity.source_full_path)
-
           {
-            resource: [resource, encoded_full_path, 'badges'].join('/'),
+            resource: [context.entity.base_resource_path, 'badges'].join('/'),
             query: {
               page: context.tracker.next_page
             }
diff --git a/lib/bulk_imports/features.rb b/lib/bulk_imports/features.rb
new file mode 100644
index 00000000000..952e8e62d71
--- /dev/null
+++ b/lib/bulk_imports/features.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Features
+    def self.enabled?
+      ::Feature.enabled?(:bulk_import)
+    end
+
+    def self.project_migration_enabled?(destination_namespace = nil)
+      if destination_namespace.present?
+        root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor
+
+        ::Feature.enabled?(:bulk_import_projects, root_ancestor)
+      else
+        ::Feature.enabled?(:bulk_import_projects)
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
index 1140beef48c..029b6f0f729 100644
--- a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
+++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
@@ -5,10 +5,8 @@ module BulkImports
     module Extractors
       class SubgroupsExtractor
         def extract(context)
-          encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path)
-
           response = http_client(context.configuration)
-            .each_page(:get, "groups/#{encoded_parent_path}/subgroups")
+            .each_page(:get, "#{context.entity.base_resource_path}/subgroups")
             .flat_map(&:itself)
 
           BulkImports::Pipeline::ExtractedData.new(data: response)
diff --git a/lib/bulk_imports/groups/graphql/get_group_query.rb b/lib/bulk_imports/groups/graphql/get_group_query.rb
index 911b2b67d8c..0e73a7fb4b9 100644
--- a/lib/bulk_imports/groups/graphql/get_group_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_group_query.rb
@@ -14,9 +14,9 @@ module BulkImports
           <<-'GRAPHQL'
           query($full_path: ID!) {
             group(fullPath: $full_path) {
+              id
               name
               path
-              full_path: fullPath
               description
               visibility
               emails_disabled: emailsDisabled
diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb
index 3f74bbb8cce..4784baf225c 100644
--- a/lib/bulk_imports/groups/graphql/get_projects_query.rb
+++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb
@@ -20,6 +20,7 @@ module BulkImports
                   has_next_page: hasNextPage
                 }
                 nodes {
+                  id
                   name
                   full_path: fullPath
                 }
diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb
index c318675e649..026b2e55713 100644
--- a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb
+++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb
@@ -15,7 +15,8 @@ module BulkImports
             source_full_path: data['full_path'],
             destination_name: data['name'],
             destination_namespace: context.entity.group.full_path,
-            parent_id: context.entity.id
+            parent_id: context.entity.id,
+            source_xid: GlobalID.parse(data['id']).model_id
           }
         end
 
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
index 0378a9c605d..6928ce43191 100644
--- a/lib/bulk_imports/groups/stage.rb
+++ b/lib/bulk_imports/groups/stage.rb
@@ -90,13 +90,7 @@ module BulkImports
       def feature_flag_enabled?
         destination_namespace = @bulk_import_entity.destination_namespace
 
-        if destination_namespace.present?
-          root_ancestor = Namespace.find_by_full_path(destination_namespace)&.root_ancestor
-
-          ::Feature.enabled?(:bulk_import_projects, root_ancestor)
-        else
-          ::Feature.enabled?(:bulk_import_projects)
-        end
+        BulkImports::Features.project_migration_enabled?(destination_namespace)
       end
     end
   end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
index d9efcdb1ba5..83b442458dc 100644
--- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -4,6 +4,7 @@ module BulkImports
   module Groups
     module Transformers
       class GroupAttributesTransformer
+        # rubocop: disable Style/IfUnlessModifier
         def transform(context, data)
           import_entity = context.entity
 
@@ -11,74 +12,64 @@ module BulkImports
             namespace = Namespace.find_by_full_path(import_entity.destination_namespace)
           end
 
-          data
-            .then { |data| transform_name(import_entity, namespace, data) }
-            .then { |data| transform_path(import_entity, data) }
-            .then { |data| transform_full_path(data) }
-            .then { |data| transform_parent(context, import_entity, namespace, data) }
-            .then { |data| transform_visibility_level(data) }
-            .then { |data| transform_project_creation_level(data) }
-            .then { |data| transform_subgroup_creation_level(data) }
-        end
-
-        private
+          params = {
+            'name' => group_name(namespace, data),
+            'path' => import_entity.destination_slug.parameterize,
+            'description' => data['description'],
+            'lfs_enabled' => data['lfs_enabled'],
+            'emails_disabled' => data['emails_disabled'],
+            'mentions_disabled' => data['mentions_disabled'],
+            'share_with_group_lock' => data['share_with_group_lock']
+          }
 
-        def transform_name(import_entity, namespace, data)
           if namespace.present?
-            namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord
-
-            if namespace_children_names.include?(data['name'])
-              data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base|
-                namespace_children_names.include?(base)
-              end
-            end
+            params['parent_id'] = namespace.id
           end
 
-          data
-        end
-
-        def transform_path(import_entity, data)
-          data['path'] = import_entity.destination_slug.parameterize
-          data
-        end
+          if data.has_key?('two_factor_grace_period')
+            params['two_factor_grace_period'] = data['two_factor_grace_period']
+          end
 
-        def transform_full_path(data)
-          data.delete('full_path')
-          data
-        end
+          if data.has_key?('request_access_enabled')
+            params['request_access_enabled'] = data['request_access_enabled']
+          end
 
-        def transform_parent(context, import_entity, namespace, data)
-          data['parent_id'] = namespace.id if namespace.present?
+          if data.has_key?('require_two_factor_authentication')
+            params['require_two_factor_authentication'] = data['require_two_factor_authentication']
+          end
 
-          data
-        end
+          if data.has_key?('project_creation_level')
+            params['project_creation_level'] =
+              Gitlab::Access.project_creation_string_options[data['project_creation_level']]
+          end
 
-        def transform_visibility_level(data)
-          visibility = data['visibility']
+          if data.has_key?('subgroup_creation_level')
+            params['subgroup_creation_level'] =
+              Gitlab::Access.subgroup_creation_string_options[data['subgroup_creation_level']]
+          end
 
-          return data unless visibility.present?
+          if data.has_key?('visibility')
+            params['visibility_level'] = Gitlab::VisibilityLevel.string_options[data['visibility']]
+          end
 
-          data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility]
-          data.delete('visibility')
-          data
+          params
         end
+        # rubocop: enable Style/IfUnlessModifier
 
-        def transform_project_creation_level(data)
-          project_creation_level = data['project_creation_level']
-
-          return data unless project_creation_level.present?
-
-          data['project_creation_level'] = Gitlab::Access.project_creation_string_options[project_creation_level]
-          data
-        end
+        private
 
-        def transform_subgroup_creation_level(data)
-          subgroup_creation_level = data['subgroup_creation_level']
+        def group_name(namespace, data)
+          if namespace.present?
+            namespace_children_names = namespace.children.pluck(:name) # rubocop: disable CodeReuse/ActiveRecord
 
-          return data unless subgroup_creation_level.present?
+            if namespace_children_names.include?(data['name'])
+              data['name'] = Uniquify.new(1).string(-> (counter) { "#{data['name']}(#{counter})" }) do |base|
+                namespace_children_names.include?(base)
+              end
+            end
+          end
 
-          data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level]
-          data
+          data['name']
         end
       end
     end
diff --git a/lib/bulk_imports/network_error.rb b/lib/bulk_imports/network_error.rb
index 3514291a75d..fda4bb74a30 100644
--- a/lib/bulk_imports/network_error.rb
+++ b/lib/bulk_imports/network_error.rb
@@ -2,15 +2,21 @@
 
 module BulkImports
   class NetworkError < Error
-    COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}'
+    TRACKER_COUNTER_KEY = 'bulk_imports/%{entity_id}/%{stage}/%{tracker_id}/network_error/%{error}'
+    ENTITY_COUNTER_KEY = 'bulk_imports/%{entity_id}/network_error/%{error}'
 
-    RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS
+    RETRIABLE_EXCEPTIONS = Gitlab::HTTP::HTTP_TIMEOUT_ERRORS + [
+      EOFError, SocketError, OpenSSL::SSL::SSLError, OpenSSL::OpenSSLError,
+      Errno::ECONNRESET, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ENETUNREACH
+    ].freeze
     RETRIABLE_HTTP_CODES = [429].freeze
 
     DEFAULT_RETRY_DELAY_SECONDS = 30
 
     MAX_RETRIABLE_COUNT = 10
 
+    attr_reader :response
+
     def initialize(message = nil, response: nil)
       raise ArgumentError, 'message or response required' if message.blank? && response.blank?
 
@@ -19,9 +25,9 @@ module BulkImports
       @response = response
     end
 
-    def retriable?(tracker)
+    def retriable?(object)
       if retriable_exception? || retriable_http_code?
-        increment(tracker) <= MAX_RETRIABLE_COUNT
+        increment(object) <= MAX_RETRIABLE_COUNT
       else
         false
       end
@@ -37,8 +43,6 @@ module BulkImports
 
     private
 
-    attr_reader :response
-
     def retriable_exception?
       RETRIABLE_EXCEPTIONS.include?(cause&.class)
     end
@@ -47,15 +51,27 @@ module BulkImports
       RETRIABLE_HTTP_CODES.include?(response&.code)
     end
 
-    def increment(tracker)
-      key = COUNTER_KEY % {
+    def increment(object)
+      key = object.is_a?(BulkImports::Entity) ? entity_cache_key(object) : tracker_cache_key(object)
+
+      Gitlab::Cache::Import::Caching.increment(key)
+    end
+
+    def tracker_cache_key(tracker)
+      TRACKER_COUNTER_KEY % {
         stage: tracker.stage,
         tracker_id: tracker.id,
         entity_id: tracker.entity.id,
         error: cause.class.name
       }
+    end
 
-      Gitlab::Cache::Import::Caching.increment(key)
+    def entity_cache_key(entity)
+      ENTITY_COUNTER_KEY % {
+        import_id: entity.bulk_import_id,
+        entity_id: entity.id,
+        error: cause.class.name
+      }
     end
   end
 end
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index 68b86c68619..54d5d3209f8 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -183,7 +183,11 @@ module BulkImports
       end
 
       def relation
-        class_attributes[:relation_name]
+        class_attributes[:relation_name] || default_relation
+      end
+
+      def default_relation
+        self.name.demodulize.chomp('Pipeline').underscore
       end
 
       private
diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb
index d753f888671..fde24cf3646 100644
--- a/lib/bulk_imports/pipeline/context.rb
+++ b/lib/bulk_imports/pipeline/context.rb
@@ -32,6 +32,10 @@ module BulkImports
         @bulk_import ||= entity.bulk_import
       end
 
+      def bulk_import_id
+        @bulk_import_id ||= bulk_import.id
+      end
+
       def current_user
         @current_user ||= bulk_import.user
       end
diff --git a/lib/bulk_imports/pipeline/runner.rb b/lib/bulk_imports/pipeline/runner.rb
index c03da7d8d01..ef9575d1e96 100644
--- a/lib/bulk_imports/pipeline/runner.rb
+++ b/lib/bulk_imports/pipeline/runner.rb
@@ -54,7 +54,8 @@ module BulkImports
         skip!(
           'Skipping pipeline due to failed entity',
           pipeline_step: step,
-          step_class: class_name
+          step_class: class_name,
+          importer: 'gitlab_migration'
         )
       rescue BulkImports::NetworkError => e
         if e.retriable?(context.tracker)
@@ -108,9 +109,12 @@ module BulkImports
         }
 
         error(
+          bulk_import_id: context.bulk_import_id,
           pipeline_step: step,
           exception_class: exception.class.to_s,
-          exception_message: exception.message
+          exception_message: exception.message,
+          message: "Pipeline failed",
+          importer: 'gitlab_migration'
         )
 
         BulkImports::Failure.create(attributes)
@@ -130,11 +134,12 @@ module BulkImports
 
       def log_params(extra)
         defaults = {
-          bulk_import_id: context.bulk_import.id,
+          bulk_import_id: context.bulk_import_id,
           bulk_import_entity_id: context.entity.id,
           bulk_import_entity_type: context.entity.source_type,
           pipeline_class: pipeline,
-          context_extra: context.extra
+          context_extra: context.extra,
+          importer: 'gitlab_migration'
         }
 
         defaults
diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb
index 76475893ac1..a2d7094d570 100644
--- a/lib/bulk_imports/projects/graphql/get_project_query.rb
+++ b/lib/bulk_imports/projects/graphql/get_project_query.rb
@@ -10,6 +10,7 @@ module BulkImports
           <<-'GRAPHQL'
           query($full_path: ID!) {
             project(fullPath: $full_path) {
+              id
               visibility
               created_at: createdAt
             }