Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42

10 files changed, 275 insertions, 53 deletions

diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb
index 1a709179bf9..aa9221cceee 100644
--- a/lib/bulk_imports/groups/pipelines/entity_finisher.rb
+++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 module BulkImports
-  module Groups
+  module Common
     module Pipelines
       class EntityFinisher
         def self.ndjson_pipeline?
diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb
new file mode 100644
index 00000000000..4cec1ad1462
--- /dev/null
+++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Graphql
+      module GetProjectsQuery
+        extend self
+
+        def to_s
+          <<-'GRAPHQL'
+          query($full_path: ID!, $cursor: String, $per_page: Int) {
+            group(fullPath: $full_path) {
+              projects(includeSubgroups: false, first: $per_page, after: $cursor) {
+                page_info: pageInfo {
+                  next_page: endCursor
+                  has_next_page: hasNextPage
+                }
+                nodes {
+                  name
+                  full_path: fullPath
+                }
+              }
+            }
+          }
+          GRAPHQL
+        end
+
+        def variables(context)
+          {
+            full_path: context.entity.source_full_path,
+            cursor: context.tracker.next_page,
+            per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE
+          }
+        end
+
+        def base_path
+          %w[data group projects]
+        end
+
+        def data_path
+          base_path << 'nodes'
+        end
+
+        def page_info_path
+          base_path << 'page_info'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb
new file mode 100644
index 00000000000..c318675e649
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Pipelines
+      class ProjectEntitiesPipeline
+        include Pipeline
+
+        extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectsQuery
+        transformer Common::Transformers::ProhibitedAttributesTransformer
+
+        def transform(context, data)
+          {
+            source_type: :project_entity,
+            source_full_path: data['full_path'],
+            destination_name: data['name'],
+            destination_namespace: context.entity.group.full_path,
+            parent_id: context.entity.id
+          }
+        end
+
+        def load(context, data)
+          context.bulk_import.entities.create!(data)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb
new file mode 100644
index 00000000000..8c3b6975b73
--- /dev/null
+++ b/lib/bulk_imports/groups/stage.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    class Stage < ::BulkImports::Stage
+      private
+
+      def config
+        @config ||= {
+          group: {
+            pipeline: BulkImports::Groups::Pipelines::GroupPipeline,
+            stage: 0
+          },
+          avatar: {
+            pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline,
+            stage: 1
+          },
+          subgroups: {
+            pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
+            stage: 1
+          },
+          members: {
+            pipeline: BulkImports::Groups::Pipelines::MembersPipeline,
+            stage: 1
+          },
+          labels: {
+            pipeline: BulkImports::Groups::Pipelines::LabelsPipeline,
+            stage: 1
+          },
+          milestones: {
+            pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline,
+            stage: 1
+          },
+          badges: {
+            pipeline: BulkImports::Groups::Pipelines::BadgesPipeline,
+            stage: 1
+          },
+          boards: {
+            pipeline: BulkImports::Groups::Pipelines::BoardsPipeline,
+            stage: 2
+          },
+          finisher: {
+            pipeline: BulkImports::Common::Pipelines::EntityFinisher,
+            stage: 3
+          }
+        }.merge(project_entities_pipeline)
+      end
+
+      def project_entities_pipeline
+        if ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml)
+          {
+            project_entities: {
+              pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline,
+              stage: 1
+            }
+          }
+        else
+          {}
+        end
+      end
+    end
+  end
+end
+
+::BulkImports::Groups::Stage.prepend_mod_with('BulkImports::Groups::Stage')
diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb
index f27818dae18..6798936576b 100644
--- a/lib/bulk_imports/pipeline.rb
+++ b/lib/bulk_imports/pipeline.rb
@@ -69,8 +69,8 @@ module BulkImports
       # Multiple transformers can be defined within a single
       # pipeline and run sequentially for each record in the
       # following order:
-      #   - Transformers defined using `transformer` class method
       #   - Instance method `transform`
+      #   - Transformers defined using `transformer` class method
       #
       # Instance method `transform` is always the last to run.
       #
diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb
new file mode 100644
index 00000000000..2aec496880f
--- /dev/null
+++ b/lib/bulk_imports/projects/graphql/get_project_query.rb
@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Graphql
+      module GetProjectQuery
+        extend self
+
+        def to_s
+          <<-'GRAPHQL'
+          query($full_path: ID!) {
+            project(fullPath: $full_path) {
+              description
+              visibility
+              archived
+              created_at: createdAt
+              shared_runners_enabled: sharedRunnersEnabled
+              container_registry_enabled: containerRegistryEnabled
+              only_allow_merge_if_pipeline_succeeds: onlyAllowMergeIfPipelineSucceeds
+              only_allow_merge_if_all_discussions_are_resolved: onlyAllowMergeIfAllDiscussionsAreResolved
+              request_access_enabled: requestAccessEnabled
+              printing_merge_request_link_enabled: printingMergeRequestLinkEnabled
+              remove_source_branch_after_merge: removeSourceBranchAfterMerge
+              autoclose_referenced_issues: autocloseReferencedIssues
+              suggestion_commit_message: suggestionCommitMessage
+              wiki_enabled: wikiEnabled
+            }
+          }
+          GRAPHQL
+        end
+
+        def variables(context)
+          { full_path: context.entity.source_full_path }
+        end
+
+        def base_path
+          %w[data project]
+        end
+
+        def data_path
+          base_path
+        end
+
+        def page_info_path
+          base_path << 'page_info'
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/pipelines/project_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_pipeline.rb
new file mode 100644
index 00000000000..c9da33fe8e3
--- /dev/null
+++ b/lib/bulk_imports/projects/pipelines/project_pipeline.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Pipelines
+      class ProjectPipeline
+        include Pipeline
+
+        abort_on_failure!
+
+        extractor ::BulkImports::Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectQuery
+        transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer
+        transformer ::BulkImports::Projects::Transformers::ProjectAttributesTransformer
+
+        def load(context, data)
+          project = ::Projects::CreateService.new(context.current_user, data).execute
+
+          if project.persisted?
+            context.entity.update!(project: project)
+
+            project
+          else
+            raise(::BulkImports::Error, "Unable to import project #{project.full_path}. #{project.errors.full_messages}.")
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb
new file mode 100644
index 00000000000..b606003091b
--- /dev/null
+++ b/lib/bulk_imports/projects/stage.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    class Stage < ::BulkImports::Stage
+      private
+
+      def config
+        @config ||= {
+          group: {
+            pipeline: BulkImports::Projects::Pipelines::ProjectPipeline,
+            stage: 0
+          },
+          finisher: {
+            pipeline: BulkImports::Common::Pipelines::EntityFinisher,
+            stage: 1
+          }
+        }
+      end
+    end
+  end
+end
+
+::BulkImports::Projects::Stage.prepend_mod_with('BulkImports::Projects::Stage')
diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb
new file mode 100644
index 00000000000..24c55d8dbb1
--- /dev/null
+++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Projects
+    module Transformers
+      class ProjectAttributesTransformer
+        PROJECT_IMPORT_TYPE = 'gitlab_project_migration'
+
+        def transform(context, data)
+          entity = context.entity
+          visibility = data.delete('visibility')
+
+          data['name'] = entity.destination_name
+          data['path'] = entity.destination_name.parameterize
+          data['import_type'] = PROJECT_IMPORT_TYPE
+          data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] if visibility.present?
+          data['namespace_id'] = Namespace.find_by_full_path(entity.destination_namespace)&.id if entity.destination_namespace.present?
+
+          data.transform_keys!(&:to_sym)
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb
index b1bceecbaea..103623cd030 100644
--- a/lib/bulk_imports/stage.rb
+++ b/lib/bulk_imports/stage.rb
@@ -2,55 +2,8 @@
 
 module BulkImports
   class Stage
-    include Singleton
-
-    CONFIG = {
-      group: {
-        pipeline: BulkImports::Groups::Pipelines::GroupPipeline,
-        stage: 0
-      },
-      avatar: {
-        pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline,
-        stage: 1
-      },
-      subgroups: {
-        pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline,
-        stage: 1
-      },
-      members: {
-        pipeline: BulkImports::Groups::Pipelines::MembersPipeline,
-        stage: 1
-      },
-      labels: {
-        pipeline: BulkImports::Groups::Pipelines::LabelsPipeline,
-        stage: 1
-      },
-      milestones: {
-        pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline,
-        stage: 1
-      },
-      badges: {
-        pipeline: BulkImports::Groups::Pipelines::BadgesPipeline,
-        stage: 1
-      },
-      boards: {
-        pipeline: BulkImports::Groups::Pipelines::BoardsPipeline,
-        stage: 2
-      },
-      finisher: {
-        pipeline: BulkImports::Groups::Pipelines::EntityFinisher,
-        stage: 3
-      }
-    }.freeze
-
     def self.pipelines
-      instance.pipelines
-    end
-
-    def self.pipeline_exists?(name)
-      pipelines.any? do |(_, pipeline)|
-        pipeline.to_s == name.to_s
-      end
+      new.pipelines
     end
 
     def pipelines
@@ -65,9 +18,8 @@ module BulkImports
     private
 
     def config
-      @config ||= CONFIG
+      # To be implemented in a sub-class
+      NotImplementedError
     end
   end
 end
-
-::BulkImports::Stage.prepend_mod_with('BulkImports::Stage')