Add latest changes from gitlab-org/gitlab@master

11 files changed, 160 insertions, 25 deletions

diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb
index 39f56fcc114..2e81863e53a 100644
--- a/lib/bulk_imports/clients/http.rb
+++ b/lib/bulk_imports/clients/http.rb
@@ -18,7 +18,7 @@ module BulkImports
       end
 
       def get(resource, query = {})
-        response = with_error_handling do
+        with_error_handling do
           Gitlab::HTTP.get(
             resource_url(resource),
             headers: request_headers,
@@ -26,8 +26,22 @@ module BulkImports
             query: query.merge(request_query)
           )
         end
+      end
+
+      def each_page(method, resource, query = {}, &block)
+        return to_enum(__method__, method, resource, query) unless block_given?
+
+        next_page = @page
 
-        response.parsed_response
+        while next_page
+          @page = next_page.to_i
+
+          response = self.public_send(method, resource, query) # rubocop: disable GitlabSecurity/PublicSend
+          collection = response.parsed_response
+          next_page = response.headers['x-next-page'].presence
+
+          yield collection
+        end
       end
 
       private
diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb
index 571be747dca..7d58032cfcc 100644
--- a/lib/bulk_imports/common/extractors/graphql_extractor.rb
+++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb
@@ -14,11 +14,9 @@ module BulkImports
           @context = context
 
           Enumerator.new do |yielder|
-            context.entities.each do |entity|
-              result = graphql_client.execute(parsed_query, query_variables(entity))
+            result = graphql_client.execute(parsed_query, query_variables(context.entity))
 
-              yielder << result.original_hash.deep_dup
-            end
+            yielder << result.original_hash.deep_dup
           end
         end
 
diff --git a/lib/bulk_imports/common/loaders/entities_loader.rb b/lib/bulk_imports/common/loaders/entities_loader.rb
new file mode 100644
index 00000000000..19926fa9a3e
--- /dev/null
+++ b/lib/bulk_imports/common/loaders/entities_loader.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Common
+    module Loaders
+      class EntitiesLoader
+        def initialize(*args); end
+
+        def load(context, entities)
+          bulk_import = context.entity.bulk_import
+
+          entities.each do |entity|
+            bulk_import.entities.create!(entity)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
new file mode 100644
index 00000000000..c33bae6ada4
--- /dev/null
+++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Extractors
+      class SubgroupsExtractor
+        def initialize(*args); end
+
+        def extract(context)
+          encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path)
+
+          subgroups = []
+          http_client(context.entity.bulk_import.configuration)
+            .each_page(:get, "groups/#{encoded_parent_path}/subgroups") do |page|
+              subgroups << page
+            end
+          subgroups
+        end
+
+        private
+
+        def http_client(configuration)
+          @http_client ||= BulkImports::Clients::Http.new(
+            uri: configuration.url,
+            token: configuration.access_token,
+            per_page: 100
+          )
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb
index 394f0ee10ec..386fc695182 100644
--- a/lib/bulk_imports/groups/loaders/group_loader.rb
+++ b/lib/bulk_imports/groups/loaders/group_loader.rb
@@ -11,7 +11,11 @@ module BulkImports
         def load(context, data)
           return unless user_can_create_group?(context.current_user, data)
 
-          ::Groups::CreateService.new(context.current_user, data).execute
+          group = ::Groups::CreateService.new(context.current_user, data).execute
+
+          context.entity.update!(group: group)
+
+          group
         end
 
         private
diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
new file mode 100644
index 00000000000..30b46a3fa24
--- /dev/null
+++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Pipelines
+      class SubgroupEntitiesPipeline
+        include Pipeline
+
+        extractor BulkImports::Groups::Extractors::SubgroupsExtractor
+        transformer BulkImports::Groups::Transformers::SubgroupsToEntitiesTransformer
+        loader BulkImports::Common::Loaders::EntitiesLoader
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
index c3937cfe652..7de9a430421 100644
--- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
+++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb
@@ -9,7 +9,7 @@ module BulkImports
         end
 
         def transform(context, data)
-          import_entity = find_by_full_path(data['full_path'], context.entities)
+          import_entity = context.entity
 
           data
             .then { |data| transform_name(import_entity, data) }
@@ -75,10 +75,6 @@ module BulkImports
           data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level]
           data
         end
-
-        def find_by_full_path(full_path, entities)
-          entities.find { |entity| entity.source_full_path == full_path }
-        end
       end
     end
   end
diff --git a/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb b/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb
new file mode 100644
index 00000000000..090b3552338
--- /dev/null
+++ b/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Groups
+    module Transformers
+      class SubgroupsToEntitiesTransformer
+        def initialize(*args); end
+
+        def transform(context, data)
+          data.map do |entry|
+            {
+              source_type: :group_entity,
+              source_full_path: entry['full_path'],
+              destination_name: entry['name'],
+              destination_namespace: context.entity.group.full_path,
+              parent_id: context.entity.id
+            }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb
index f053177d9fb..c7253590c87 100644
--- a/lib/bulk_imports/importers/group_importer.rb
+++ b/lib/bulk_imports/importers/group_importer.rb
@@ -1,34 +1,32 @@
 # frozen_string_literal: true
 
-# Imports a top level group into a destination
-# Optionally imports into parent group
-# Entity must be of type: 'group' & have parent_id: nil
-# Subgroups not handled yet
 module BulkImports
   module Importers
     class GroupImporter
-      def initialize(entity_id)
-        @entity_id = entity_id
+      def initialize(entity)
+        @entity = entity
       end
 
       def execute
-        return if entity.parent
-
+        entity.start!
         bulk_import = entity.bulk_import
         configuration = bulk_import.configuration
 
         context = BulkImports::Pipeline::Context.new(
           current_user: bulk_import.user,
-          entities: [entity],
+          entity: entity,
           configuration: configuration
         )
 
         BulkImports::Groups::Pipelines::GroupPipeline.new.run(context)
-      end
+        BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context)
 
-      def entity
-        @entity ||= BulkImports::Entity.find(@entity_id)
+        entity.finish!
       end
+
+      private
+
+      attr_reader :entity
     end
   end
 end
diff --git a/lib/bulk_imports/importers/groups_importer.rb b/lib/bulk_imports/importers/groups_importer.rb
new file mode 100644
index 00000000000..8641577ff47
--- /dev/null
+++ b/lib/bulk_imports/importers/groups_importer.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module BulkImports
+  module Importers
+    class GroupsImporter
+      def initialize(bulk_import_id)
+        @bulk_import = BulkImport.find(bulk_import_id)
+      end
+
+      def execute
+        bulk_import.start! unless bulk_import.started?
+
+        if entities_to_import.empty?
+          bulk_import.finish!
+        else
+          entities_to_import.each do |entity|
+            BulkImports::Importers::GroupImporter.new(entity).execute
+          end
+
+          # A new BulkImportWorker job is enqueued to either
+          #   - Process the new BulkImports::Entity created for the subgroups
+          #   - Or to mark the `bulk_import` as finished.
+          BulkImportWorker.perform_async(bulk_import.id)
+        end
+      end
+
+      private
+
+      attr_reader :bulk_import
+
+      def entities_to_import
+        @entities_to_import ||= bulk_import.entities.with_status(:created)
+      end
+    end
+  end
+end
diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb
index 903f474ebbb..ad19f5cad7d 100644
--- a/lib/bulk_imports/pipeline/context.rb
+++ b/lib/bulk_imports/pipeline/context.rb
@@ -9,7 +9,7 @@ module BulkImports
 
       PIPELINE_ATTRIBUTES = [
         Attribute.new(:current_user, User),
-        Attribute.new(:entities, Array),
+        Attribute.new(:entity, ::BulkImports::Entity),
         Attribute.new(:configuration, ::BulkImports::Configuration)
       ].freeze