diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/bulk_imports/clients/http.rb | 2 | ||||
| -rw-r--r-- | lib/bulk_imports/common/pipelines/wiki_pipeline.rb | 21 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb | 15 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/stage.rb | 4 |
4 files changed, 37 insertions, 5 deletions
diff --git a/lib/bulk_imports/clients/http.rb b/lib/bulk_imports/clients/http.rb index 90414a875c6..eb3d551d1d7 100644 --- a/lib/bulk_imports/clients/http.rb +++ b/lib/bulk_imports/clients/http.rb @@ -123,7 +123,7 @@ module BulkImports def with_error_handling response = yield - raise ::BulkImports::NetworkError.new(response: response) unless response.success? + raise ::BulkImports::NetworkError.new("Unsuccessful response #{response.code} from #{response.request.path.path}", response: response) unless response.success? response rescue *Gitlab::HTTP::HTTP_ERRORS => e diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb index ccab0b979b2..6900835b14d 100644 --- a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -7,7 +7,9 @@ module BulkImports include Pipeline def extract(*) - BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) }) + url = url_from_parent_path(context.entity.source_full_path) if source_wiki_exists? + + BulkImports::Pipeline::ExtractedData.new(data: { url: url }) end def transform(_, data) @@ -15,14 +17,15 @@ module BulkImports end def load(context, data) - return unless context.portable.wiki + return unless data&.dig(:url) + wiki = context.portable.wiki url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@") Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) - context.portable.wiki.ensure_repository - context.portable.wiki.repository.fetch_as_mirror(url) + wiki.ensure_repository + wiki.repository.fetch_as_mirror(url) end private @@ -36,6 +39,16 @@ module BulkImports def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end + + def source_wiki_exists? + wikis = client.get(context.entity.wikis_url_path).parsed_response + + wikis.any? + end + + def client + BulkImports::Clients::HTTP.new(url: context.configuration.url, token: context.configuration.access_token) + end end end end diff --git a/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb new file mode 100644 index 00000000000..1e54ca7017d --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/auto_devops_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class AutoDevopsPipeline + include NdjsonPipeline + + relation_name 'auto_devops' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index d95130b59e0..e16e4369689 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -67,6 +67,10 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::UploadsPipeline, stage: 5 }, + auto_devops: { + pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, + stage: 5 + }, finisher: { pipeline: BulkImports::Common::Pipelines::EntityFinisher, stage: 6 |