diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-06 12:08:56 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2020-11-06 12:08:56 +0300 |
commit | 0dce1c285f8d6487daf4b83be1ca9585e3a084e6 (patch) | |
tree | f617aa00d3994c2733baaed0205dba0d3bc0413d /lib | |
parent | eefbee4451565989727256d36176dc2950e3a0b7 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'lib')
11 files changed, 160 insertions, 25 deletions
diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 39f56fcc114..2e81863e53a 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -18,7 +18,7 @@ module BulkImports end def get(resource, query = {}) - response = with_error_handling do + with_error_handling do Gitlab::HTTP.get( resource_url(resource), headers: request_headers, @@ -26,8 +26,22 @@ module BulkImports query: query.merge(request_query) ) end + end + + def each_page(method, resource, query = {}, &block) + return to_enum(__method__, method, resource, query) unless block_given? + + next_page = @page - response.parsed_response + while next_page + @page = next_page.to_i + + response = self.public_send(method, resource, query) # rubocop: disable GitlabSecurity/PublicSend + collection = response.parsed_response + next_page = response.headers['x-next-page'].presence + + yield collection + end end private diff --git a/lib/bulk_imports/common/extractors/graphql_extractor.rb b/lib/bulk_imports/common/extractors/graphql_extractor.rb index 571be747dca..7d58032cfcc 100644 --- a/lib/bulk_imports/common/extractors/graphql_extractor.rb +++ b/lib/bulk_imports/common/extractors/graphql_extractor.rb @@ -14,11 +14,9 @@ module BulkImports @context = context Enumerator.new do |yielder| - context.entities.each do |entity| - result = graphql_client.execute(parsed_query, query_variables(entity)) + result = graphql_client.execute(parsed_query, query_variables(context.entity)) - yielder << result.original_hash.deep_dup - end + yielder << result.original_hash.deep_dup end end diff --git a/lib/bulk_imports/common/loaders/entities_loader.rb b/lib/bulk_imports/common/loaders/entities_loader.rb new file mode 100644 index 00000000000..19926fa9a3e --- /dev/null +++ b/lib/bulk_imports/common/loaders/entities_loader.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Loaders + class EntitiesLoader + def initialize(*args); end + + def load(context, entities) + bulk_import = context.entity.bulk_import + + entities.each do |entity| + bulk_import.entities.create!(entity) + end + end + end + end + end +end diff --git a/lib/bulk_imports/groups/extractors/subgroups_extractor.rb b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb new file mode 100644 index 00000000000..c33bae6ada4 --- /dev/null +++ b/lib/bulk_imports/groups/extractors/subgroups_extractor.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Extractors + class SubgroupsExtractor + def initialize(*args); end + + def extract(context) + encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path) + + subgroups = [] + http_client(context.entity.bulk_import.configuration) + .each_page(:get, "groups/#{encoded_parent_path}/subgroups") do |page| + subgroups << page + end + subgroups + end + + private + + def http_client(configuration) + @http_client ||= BulkImports::Clients::Http.new( + uri: configuration.url, + token: configuration.access_token, + per_page: 100 + ) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/loaders/group_loader.rb b/lib/bulk_imports/groups/loaders/group_loader.rb index 394f0ee10ec..386fc695182 100644 --- a/lib/bulk_imports/groups/loaders/group_loader.rb +++ b/lib/bulk_imports/groups/loaders/group_loader.rb @@ -11,7 +11,11 @@ module BulkImports def load(context, data) return unless user_can_create_group?(context.current_user, data) - ::Groups::CreateService.new(context.current_user, data).execute + group = ::Groups::CreateService.new(context.current_user, data).execute + + context.entity.update!(group: group) + + group end private diff --git a/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb new file mode 100644 index 00000000000..30b46a3fa24 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/subgroup_entities_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class SubgroupEntitiesPipeline + include Pipeline + + extractor BulkImports::Groups::Extractors::SubgroupsExtractor + transformer BulkImports::Groups::Transformers::SubgroupsToEntitiesTransformer + loader BulkImports::Common::Loaders::EntitiesLoader + end + end + end +end diff --git a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb index c3937cfe652..7de9a430421 100644 --- a/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb +++ b/lib/bulk_imports/groups/transformers/group_attributes_transformer.rb @@ -9,7 +9,7 @@ module BulkImports end def transform(context, data) - import_entity = find_by_full_path(data['full_path'], context.entities) + import_entity = context.entity data .then { |data| transform_name(import_entity, data) } @@ -75,10 +75,6 @@ module BulkImports data['subgroup_creation_level'] = Gitlab::Access.subgroup_creation_string_options[subgroup_creation_level] data end - - def find_by_full_path(full_path, entities) - entities.find { |entity| entity.source_full_path == full_path } - end end end end diff --git a/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb b/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb new file mode 100644 index 00000000000..090b3552338 --- /dev/null +++ b/lib/bulk_imports/groups/transformers/subgroups_to_entities_transformer.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Transformers + class SubgroupsToEntitiesTransformer + def initialize(*args); end + + def transform(context, data) + data.map do |entry| + { + source_type: :group_entity, + source_full_path: entry['full_path'], + destination_name: entry['name'], + destination_namespace: context.entity.group.full_path, + parent_id: context.entity.id + } + end + end + end + end + end +end diff --git a/lib/bulk_imports/importers/group_importer.rb b/lib/bulk_imports/importers/group_importer.rb index f053177d9fb..c7253590c87 100644 --- a/lib/bulk_imports/importers/group_importer.rb +++ b/lib/bulk_imports/importers/group_importer.rb @@ -1,34 +1,32 @@ # frozen_string_literal: true -# Imports a top level group into a destination -# Optionally imports into parent group -# Entity must be of type: 'group' & have parent_id: nil -# Subgroups not handled yet module BulkImports module Importers class GroupImporter - def initialize(entity_id) - @entity_id = entity_id + def initialize(entity) + @entity = entity end def execute - return if entity.parent - + entity.start! bulk_import = entity.bulk_import configuration = bulk_import.configuration context = BulkImports::Pipeline::Context.new( current_user: bulk_import.user, - entities: [entity], + entity: entity, configuration: configuration ) BulkImports::Groups::Pipelines::GroupPipeline.new.run(context) - end + BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline.new.run(context) - def entity - @entity ||= BulkImports::Entity.find(@entity_id) + entity.finish! end + + private + + attr_reader :entity end end end diff --git a/lib/bulk_imports/importers/groups_importer.rb b/lib/bulk_imports/importers/groups_importer.rb new file mode 100644 index 00000000000..8641577ff47 --- /dev/null +++ b/lib/bulk_imports/importers/groups_importer.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module BulkImports + module Importers + class GroupsImporter + def initialize(bulk_import_id) + @bulk_import = BulkImport.find(bulk_import_id) + end + + def execute + bulk_import.start! unless bulk_import.started? + + if entities_to_import.empty? + bulk_import.finish! + else + entities_to_import.each do |entity| + BulkImports::Importers::GroupImporter.new(entity).execute + end + + # A new BulkImportWorker job is enqueued to either + # - Process the new BulkImports::Entity created for the subgroups + # - Or to mark the `bulk_import` as finished. + BulkImportWorker.perform_async(bulk_import.id) + end + end + + private + + attr_reader :bulk_import + + def entities_to_import + @entities_to_import ||= bulk_import.entities.with_status(:created) + end + end + end +end diff --git a/lib/bulk_imports/pipeline/context.rb b/lib/bulk_imports/pipeline/context.rb index 903f474ebbb..ad19f5cad7d 100644 --- a/lib/bulk_imports/pipeline/context.rb +++ b/lib/bulk_imports/pipeline/context.rb @@ -9,7 +9,7 @@ module BulkImports PIPELINE_ATTRIBUTES = [ Attribute.new(:current_user, User), - Attribute.new(:entities, Array), + Attribute.new(:entity, ::BulkImports::Entity), Attribute.new(:configuration, ::BulkImports::Configuration) ].freeze |