diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 16:18:24 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-09-20 16:18:24 +0300 |
commit | 0653e08efd039a5905f3fa4f6e9cef9f5d2f799c (patch) | |
tree | 4dcc884cf6d81db44adae4aa99f8ec1233a41f55 /lib/bulk_imports | |
parent | 744144d28e3e7fddc117924fef88de5d9674fe4c (diff) |
Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42
Diffstat (limited to 'lib/bulk_imports')
-rw-r--r-- | lib/bulk_imports/common/pipelines/entity_finisher.rb (renamed from lib/bulk_imports/groups/pipelines/entity_finisher.rb) | 2 | ||||
-rw-r--r-- | lib/bulk_imports/groups/graphql/get_projects_query.rb | 50 | ||||
-rw-r--r-- | lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb | 28 | ||||
-rw-r--r-- | lib/bulk_imports/groups/stage.rb | 65 | ||||
-rw-r--r-- | lib/bulk_imports/pipeline.rb | 2 | ||||
-rw-r--r-- | lib/bulk_imports/projects/graphql/get_project_query.rb | 50 | ||||
-rw-r--r-- | lib/bulk_imports/projects/pipelines/project_pipeline.rb | 29 | ||||
-rw-r--r-- | lib/bulk_imports/projects/stage.rb | 24 | ||||
-rw-r--r-- | lib/bulk_imports/projects/transformers/project_attributes_transformer.rb | 24 | ||||
-rw-r--r-- | lib/bulk_imports/stage.rb | 54 |
10 files changed, 275 insertions, 53 deletions
diff --git a/lib/bulk_imports/groups/pipelines/entity_finisher.rb b/lib/bulk_imports/common/pipelines/entity_finisher.rb index 1a709179bf9..aa9221cceee 100644 --- a/lib/bulk_imports/groups/pipelines/entity_finisher.rb +++ b/lib/bulk_imports/common/pipelines/entity_finisher.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module BulkImports - module Groups + module Common module Pipelines class EntityFinisher def self.ndjson_pipeline? diff --git a/lib/bulk_imports/groups/graphql/get_projects_query.rb b/lib/bulk_imports/groups/graphql/get_projects_query.rb new file mode 100644 index 00000000000..4cec1ad1462 --- /dev/null +++ b/lib/bulk_imports/groups/graphql/get_projects_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Graphql + module GetProjectsQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!, $cursor: String, $per_page: Int) { + group(fullPath: $full_path) { + projects(includeSubgroups: false, first: $per_page, after: $cursor) { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + name + full_path: fullPath + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data group projects] + end + + def data_path + base_path << 'nodes' + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb new file mode 100644 index 00000000000..c318675e649 --- /dev/null +++ b/lib/bulk_imports/groups/pipelines/project_entities_pipeline.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + module Pipelines + class ProjectEntitiesPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectsQuery + transformer Common::Transformers::ProhibitedAttributesTransformer + + def transform(context, data) + { + source_type: :project_entity, + source_full_path: data['full_path'], + destination_name: data['name'], + destination_namespace: context.entity.group.full_path, + parent_id: context.entity.id + } + end + + def load(context, data) + context.bulk_import.entities.create!(data) + end + end + end + end +end diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb new file mode 100644 index 00000000000..8c3b6975b73 --- /dev/null +++ b/lib/bulk_imports/groups/stage.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module BulkImports + module Groups + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Groups::Pipelines::GroupPipeline, + stage: 0 + }, + avatar: { + pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, + stage: 1 + }, + subgroups: { + pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, + stage: 1 + }, + members: { + pipeline: BulkImports::Groups::Pipelines::MembersPipeline, + stage: 1 + }, + labels: { + pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, + stage: 1 + }, + milestones: { + pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, + stage: 1 + }, + badges: { + pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, + stage: 1 + }, + boards: { + pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, + stage: 2 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 3 + } + }.merge(project_entities_pipeline) + end + + def project_entities_pipeline + if ::Feature.enabled?(:bulk_import_projects, default_enabled: :yaml) + { + project_entities: { + pipeline: BulkImports::Groups::Pipelines::ProjectEntitiesPipeline, + stage: 1 + } + } + else + {} + end + end + end + end +end + +::BulkImports::Groups::Stage.prepend_mod_with('BulkImports::Groups::Stage') diff --git a/lib/bulk_imports/pipeline.rb b/lib/bulk_imports/pipeline.rb index f27818dae18..6798936576b 100644 --- a/lib/bulk_imports/pipeline.rb +++ b/lib/bulk_imports/pipeline.rb @@ -69,8 +69,8 @@ module BulkImports # Multiple transformers can be defined within a single # pipeline and run sequentially for each record in the # following order: - # - Transformers defined using `transformer` class method # - Instance method `transform` + # - Transformers defined using `transformer` class method # # Instance method `transform` is always the last to run. # diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb new file mode 100644 index 00000000000..2aec496880f --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetProjectQuery + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + description + visibility + archived + created_at: createdAt + shared_runners_enabled: sharedRunnersEnabled + container_registry_enabled: containerRegistryEnabled + only_allow_merge_if_pipeline_succeeds: onlyAllowMergeIfPipelineSucceeds + only_allow_merge_if_all_discussions_are_resolved: onlyAllowMergeIfAllDiscussionsAreResolved + request_access_enabled: requestAccessEnabled + printing_merge_request_link_enabled: printingMergeRequestLinkEnabled + remove_source_branch_after_merge: removeSourceBranchAfterMerge + autoclose_referenced_issues: autocloseReferencedIssues + suggestion_commit_message: suggestionCommitMessage + wiki_enabled: wikiEnabled + } + } + GRAPHQL + end + + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_pipeline.rb new file mode 100644 index 00000000000..c9da33fe8e3 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_pipeline.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectPipeline + include Pipeline + + abort_on_failure! + + extractor ::BulkImports::Common::Extractors::GraphqlExtractor, query: Graphql::GetProjectQuery + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + transformer ::BulkImports::Projects::Transformers::ProjectAttributesTransformer + + def load(context, data) + project = ::Projects::CreateService.new(context.current_user, data).execute + + if project.persisted? + context.entity.update!(project: project) + + project + else + raise(::BulkImports::Error, "Unable to import project #{project.full_path}. #{project.errors.full_messages}.") + end + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb new file mode 100644 index 00000000000..b606003091b --- /dev/null +++ b/lib/bulk_imports/projects/stage.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + class Stage < ::BulkImports::Stage + private + + def config + @config ||= { + group: { + pipeline: BulkImports::Projects::Pipelines::ProjectPipeline, + stage: 0 + }, + finisher: { + pipeline: BulkImports::Common::Pipelines::EntityFinisher, + stage: 1 + } + } + end + end + end +end + +::BulkImports::Projects::Stage.prepend_mod_with('BulkImports::Projects::Stage') diff --git a/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb new file mode 100644 index 00000000000..24c55d8dbb1 --- /dev/null +++ b/lib/bulk_imports/projects/transformers/project_attributes_transformer.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Transformers + class ProjectAttributesTransformer + PROJECT_IMPORT_TYPE = 'gitlab_project_migration' + + def transform(context, data) + entity = context.entity + visibility = data.delete('visibility') + + data['name'] = entity.destination_name + data['path'] = entity.destination_name.parameterize + data['import_type'] = PROJECT_IMPORT_TYPE + data['visibility_level'] = Gitlab::VisibilityLevel.string_options[visibility] if visibility.present? + data['namespace_id'] = Namespace.find_by_full_path(entity.destination_namespace)&.id if entity.destination_namespace.present? + + data.transform_keys!(&:to_sym) + end + end + end + end +end diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index b1bceecbaea..103623cd030 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -2,55 +2,8 @@ module BulkImports class Stage - include Singleton - - CONFIG = { - group: { - pipeline: BulkImports::Groups::Pipelines::GroupPipeline, - stage: 0 - }, - avatar: { - pipeline: BulkImports::Groups::Pipelines::GroupAvatarPipeline, - stage: 1 - }, - subgroups: { - pipeline: BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline, - stage: 1 - }, - members: { - pipeline: BulkImports::Groups::Pipelines::MembersPipeline, - stage: 1 - }, - labels: { - pipeline: BulkImports::Groups::Pipelines::LabelsPipeline, - stage: 1 - }, - milestones: { - pipeline: BulkImports::Groups::Pipelines::MilestonesPipeline, - stage: 1 - }, - badges: { - pipeline: BulkImports::Groups::Pipelines::BadgesPipeline, - stage: 1 - }, - boards: { - pipeline: BulkImports::Groups::Pipelines::BoardsPipeline, - stage: 2 - }, - finisher: { - pipeline: BulkImports::Groups::Pipelines::EntityFinisher, - stage: 3 - } - }.freeze - def self.pipelines - instance.pipelines - end - - def self.pipeline_exists?(name) - pipelines.any? do |(_, pipeline)| - pipeline.to_s == name.to_s - end + new.pipelines end def pipelines @@ -65,9 +18,8 @@ module BulkImports private def config - @config ||= CONFIG + # To be implemented in a sub-class + NotImplementedError end end end - -::BulkImports::Stage.prepend_mod_with('BulkImports::Stage') |