diff options
Diffstat (limited to 'lib/bulk_imports/common')
4 files changed, 112 insertions, 21 deletions
diff --git a/lib/bulk_imports/common/extractors/ndjson_extractor.rb b/lib/bulk_imports/common/extractors/ndjson_extractor.rb index 6b4acd45ea9..ecd7c08bd25 100644 --- a/lib/bulk_imports/common/extractors/ndjson_extractor.rb +++ b/lib/bulk_imports/common/extractors/ndjson_extractor.rb @@ -7,10 +7,6 @@ module BulkImports include Gitlab::ImportExport::CommandLineUtil include Gitlab::Utils::StrongMemoize - FILE_SIZE_LIMIT = 5.gigabytes - ALLOWED_CONTENT_TYPES = %w(application/gzip application/octet-stream).freeze - EXPORT_DOWNLOAD_URL_PATH = "/%{resource}/%{full_path}/export_relations/download?relation=%{relation}" - def initialize(relation:) @relation = relation @tmp_dir = Dir.mktmpdir @@ -39,33 +35,19 @@ module BulkImports def download_service(tmp_dir, context) @download_service ||= BulkImports::FileDownloadService.new( configuration: context.configuration, - relative_url: relative_resource_url(context), + relative_url: context.entity.relation_download_url_path(relation), dir: tmp_dir, - filename: filename, - file_size_limit: FILE_SIZE_LIMIT, - allowed_content_types: ALLOWED_CONTENT_TYPES + filename: filename ) end def decompression_service(tmp_dir) - @decompression_service ||= BulkImports::FileDecompressionService.new( - dir: tmp_dir, - filename: filename - ) + @decompression_service ||= BulkImports::FileDecompressionService.new(dir: tmp_dir, filename: filename) end def ndjson_reader(tmp_dir) @ndjson_reader ||= Gitlab::ImportExport::Json::NdjsonReader.new(tmp_dir) end - - def relative_resource_url(context) - strong_memoize(:relative_resource_url) do - resource = context.entity.pluralized_name - encoded_full_path = context.entity.encoded_source_full_path - - EXPORT_DOWNLOAD_URL_PATH % { resource: resource, full_path: encoded_full_path, relation: relation } - end - end end end end diff --git a/lib/bulk_imports/common/pipelines/milestones_pipeline.rb b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb new file mode 100644 index 00000000000..aea2a04c1c7 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/milestones_pipeline.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class MilestonesPipeline + include NdjsonPipeline + + relation_name 'milestones' + + extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/uploads_pipeline.rb b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb new file mode 100644 index 00000000000..15e126e1646 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/uploads_pipeline.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class UploadsPipeline + include Pipeline + include Gitlab::ImportExport::CommandLineUtil + + FILENAME = 'uploads.tar.gz' + + def extract(context) + download_service(tmp_dir, context).execute + untar_zxf(archive: File.join(tmp_dir, FILENAME), dir: tmp_dir) + upload_file_paths = Dir.glob(File.join(tmp_dir, '**', '*')) + + BulkImports::Pipeline::ExtractedData.new(data: upload_file_paths) + end + + def load(context, file_path) + dynamic_path = FileUploader.extract_dynamic_path(file_path) + + return unless dynamic_path + return if File.directory?(file_path) + + named_captures = dynamic_path.named_captures.symbolize_keys + + UploadService.new(context.portable, File.open(file_path, 'r'), FileUploader, **named_captures).execute + end + + def after_run(_) + FileUtils.remove_entry(tmp_dir) + end + + private + + def download_service(tmp_dir, context) + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path('uploads'), + dir: tmp_dir, + filename: FILENAME + ) + end + + def tmp_dir + @tmp_dir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/common/pipelines/wiki_pipeline.rb b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb new file mode 100644 index 00000000000..ccab0b979b2 --- /dev/null +++ b/lib/bulk_imports/common/pipelines/wiki_pipeline.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module BulkImports + module Common + module Pipelines + class WikiPipeline + include Pipeline + + def extract(*) + BulkImports::Pipeline::ExtractedData.new(data: { url: url_from_parent_path(context.entity.source_full_path) }) + end + + def transform(_, data) + data&.slice(:url) + end + + def load(context, data) + return unless context.portable.wiki + + url = data[:url].sub("://", "://oauth2:#{context.configuration.access_token}@") + + Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, allow_localhost: allow_local_requests?) + + context.portable.wiki.ensure_repository + context.portable.wiki.repository.fetch_as_mirror(url) + end + + private + + def url_from_parent_path(parent_path) + wiki_path = parent_path + ".wiki.git" + root = context.configuration.url + Gitlab::Utils.append_path(root, wiki_path) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + end + end + end +end |