diff options
Diffstat (limited to 'lib/bulk_imports/projects')
15 files changed, 376 insertions, 32 deletions
diff --git a/lib/bulk_imports/projects/graphql/get_project_query.rb b/lib/bulk_imports/projects/graphql/get_project_query.rb index 2aec496880f..04ac0916bbc 100644 --- a/lib/bulk_imports/projects/graphql/get_project_query.rb +++ b/lib/bulk_imports/projects/graphql/get_project_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetProjectQuery + extend Queryable extend self def to_s @@ -28,22 +29,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_repository_query.rb b/lib/bulk_imports/projects/graphql/get_repository_query.rb index d3e377c1175..24efce9e276 100644 --- a/lib/bulk_imports/projects/graphql/get_repository_query.rb +++ b/lib/bulk_imports/projects/graphql/get_repository_query.rb @@ -4,6 +4,7 @@ module BulkImports module Projects module Graphql module GetRepositoryQuery + extend Queryable extend self def to_s @@ -15,22 +16,6 @@ module BulkImports } GRAPHQL end - - def variables(context) - { full_path: context.entity.source_full_path } - end - - def base_path - %w[data project] - end - - def data_path - base_path - end - - def page_info_path - base_path << 'page_info' - end end end end diff --git a/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb new file mode 100644 index 00000000000..1ba57789612 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/get_snippet_repository_query.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module GetSnippetRepositoryQuery + extend Queryable + extend self + + def to_s + <<-'GRAPHQL' + query($full_path: ID!) { + project(fullPath: $full_path) { + snippets { + page_info: pageInfo { + next_page: endCursor + has_next_page: hasNextPage + } + nodes { + title + createdAt + httpUrlToRepo + } + } + } + } + GRAPHQL + end + + def variables(context) + { + full_path: context.entity.source_full_path, + cursor: context.tracker.next_page, + per_page: ::BulkImports::Tracker::DEFAULT_PAGE_SIZE + } + end + + def base_path + %w[data project snippets] + end + + def data_path + base_path << 'nodes' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/graphql/queryable.rb b/lib/bulk_imports/projects/graphql/queryable.rb new file mode 100644 index 00000000000..a897632dff3 --- /dev/null +++ b/lib/bulk_imports/projects/graphql/queryable.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Graphql + module Queryable + def variables(context) + { full_path: context.entity.source_full_path } + end + + def base_path + %w[data project] + end + + def data_path + base_path + end + + def page_info_path + base_path << 'page_info' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb new file mode 100644 index 00000000000..1e54ca7017d --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class AutoDevopsPipeline + include NdjsonPipeline + + relation_name 'auto_devops' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb new file mode 100644 index 00000000000..4487835b88e --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/ci_pipelines_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class CiPipelinesPipeline + include NdjsonPipeline + + relation_name 'ci_pipelines' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb new file mode 100644 index 00000000000..796e2bd5293 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/container_expiration_policy_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ContainerExpirationPolicyPipeline + include NdjsonPipeline + + relation_name 'container_expiration_policy' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb new file mode 100644 index 00000000000..67053f4e0d4 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/pipeline_schedules_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class PipelineSchedulesPipeline + include NdjsonPipeline + + relation_name 'pipeline_schedules' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb new file mode 100644 index 00000000000..4d742225ff7 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -0,0 +1,85 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectAttributesPipeline + include Pipeline + + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer + + def extract(context) + download_service(tmp_dir, context).execute + decompression_service(tmp_dir).execute + project_attributes = json_decode(json_attributes) + + BulkImports::Pipeline::ExtractedData.new(data: project_attributes) + end + + def transform(_, data) + subrelations = config.portable_relations_tree.keys.map(&:to_s) + + Gitlab::ImportExport::AttributeCleaner.clean( + relation_hash: data, + relation_class: Project, + excluded_keys: config.relation_excluded_keys(:project) + ).except(*subrelations) + end + + def load(_, data) + portable.assign_attributes(data) + portable.reconcile_shared_runners_setting! + portable.drop_visibility_level! + portable.save! + end + + def after_run(_) + FileUtils.remove_entry(tmp_dir) + end + + def json_attributes + @json_attributes ||= File.read(File.join(tmp_dir, filename)) + end + + private + + def tmp_dir + @tmp_dir ||= Dir.mktmpdir + end + + def config + @config ||= BulkImports::FileTransfer.config_for(portable) + end + + def download_service(tmp_dir, context) + @download_service ||= BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(BulkImports::FileTransfer::BaseConfig::SELF_RELATION), + dir: tmp_dir, + filename: compressed_filename + ) + end + + def decompression_service(tmp_dir) + @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: compressed_filename) + end + + def compressed_filename + "#{filename}.gz" + end + + def filename + "#{BulkImports::FileTransfer::BaseConfig::SELF_RELATION}.json" + end + + def json_decode(string) + Gitlab::Json.parse(string) + rescue JSON::ParserError => e + Gitlab::ErrorTracking.log_exception(e) + + raise BulkImports::Error, 'Incorrect JSON format' + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb new file mode 100644 index 00000000000..ff5437efeef --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/project_feature_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ProjectFeaturePipeline + include NdjsonPipeline + + relation_name 'project_feature' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb index 6bbd4d0688b..f5ccc1dd922 100644 --- a/lib/bulk_imports/projects/pipelines/repository_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/repository_pipeline.rb @@ -16,6 +16,8 @@ module BulkImports def load(context, data) url = data['httpUrlToRepo'] + return unless url.present? + url = url.sub("://", "://oauth2:#{context.configuration.access_token}@") project = context.portable diff --git a/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb new file mode 100644 index 00000000000..a50b5423366 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/service_desk_setting_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class ServiceDeskSettingPipeline + include NdjsonPipeline + + relation_name 'service_desk_setting' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb new file mode 100644 index 00000000000..d543bcec383 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsPipeline + include NdjsonPipeline + + relation_name 'snippets' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb new file mode 100644 index 00000000000..6d423717a51 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/snippets_repository_pipeline.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class SnippetsRepositoryPipeline + include Pipeline + + extractor Common::Extractors::GraphqlExtractor, query: Graphql::GetSnippetRepositoryQuery + + def transform(_context, data) + data.tap do |d| + d['createdAt'] = DateTime.parse(data['createdAt']) + end + end + + def load(context, data) + return unless data['httpUrlToRepo'].present? + + oauth2_url = oauth2(data['httpUrlToRepo']) + validate_url(oauth2_url) + + matched_snippet = find_matched_snippet(data) + # Skip snippets that we couldn't find a match. Probably because more snippets were + # added after the migration had already started, namely after the SnippetsPipeline + # has already run. + return unless matched_snippet + + matched_snippet.create_repository + matched_snippet.repository.fetch_as_mirror(oauth2_url) + response = Snippets::RepositoryValidationService.new(nil, matched_snippet).execute + + # skips matched_snippet repository creation if repository is invalid + return cleanup_snippet_repository(matched_snippet) if response.error? + + Snippets::UpdateStatisticsService.new(matched_snippet).execute + end + + private + + def find_matched_snippet(data) + Snippet.find_by_project_title_trunc_created_at( + context.portable, data['title'], data['createdAt']) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def oauth2(url) + url.sub("://", "://oauth2:#{context.configuration.access_token}@") + end + + def validate_url(url) + Gitlab::UrlBlocker.validate!( + url, + allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?) + end + + def cleanup_snippet_repository(snippet) + snippet.repository.remove + snippet.snippet_repository.delete + snippet.repository.expire_exists_cache + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 9ccc9efff1d..0556395ca66 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -15,6 +15,10 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline, stage: 1 }, + project_attributes: { + pipeline: BulkImports::Projects::Pipelines::ProjectAttributesPipeline, + stage: 1 + }, labels: { pipeline: BulkImports::Common::Pipelines::LabelsPipeline, stage: 2 @@ -23,10 +27,22 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::MilestonesPipeline, stage: 2 }, + badges: { + pipeline: BulkImports::Common::Pipelines::BadgesPipeline, + stage: 2 + }, issues: { pipeline: BulkImports::Projects::Pipelines::IssuesPipeline, stage: 3 }, + snippets: { + pipeline: BulkImports::Projects::Pipelines::SnippetsPipeline, + stage: 3 + }, + snippets_repository: { + pipeline: BulkImports::Projects::Pipelines::SnippetsRepositoryPipeline, + stage: 4 + }, boards: { pipeline: BulkImports::Common::Pipelines::BoardsPipeline, stage: 4 @@ -43,6 +59,22 @@ module BulkImports pipeline: BulkImports::Projects::Pipelines::ProtectedBranchesPipeline, stage: 4 }, + ci_pipelines: { + pipeline: BulkImports::Projects::Pipelines::CiPipelinesPipeline, + stage: 4 + }, + project_feature: { + pipeline: BulkImports::Projects::Pipelines::ProjectFeaturePipeline, + stage: 4 + }, + container_expiration_policy: { + pipeline: BulkImports::Projects::Pipelines::ContainerExpirationPolicyPipeline, + stage: 4 + }, + service_desk_setting: { + pipeline: BulkImports::Projects::Pipelines::ServiceDeskSettingPipeline, + stage: 4 + }, wiki: { pipeline: BulkImports::Common::Pipelines::WikiPipeline, stage: 5 @@ -51,6 +83,14 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::UploadsPipeline, stage: 5 }, + auto_devops: { + pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, + stage: 5 + }, + pipeline_schedules: { + pipeline: BulkImports::Projects::Pipelines::PipelineSchedulesPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 6 |