diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 14:10:13 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-06-20 14:10:13 +0300 |
commit | 0ea3fcec397b69815975647f5e2aa5fe944a8486 (patch) | |
tree | 7979381b89d26011bcf9bdc989a40fcc2f1ed4ff /lib/bulk_imports/projects/pipelines | |
parent | 72123183a20411a36d607d70b12d57c484394c8e (diff) |
Add latest changes from gitlab-org/gitlab@15-1-stable-eev15.1.0-rc42
Diffstat (limited to 'lib/bulk_imports/projects/pipelines')
4 files changed, 161 insertions, 47 deletions
diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb new file mode 100644 index 00000000000..2d5231b0541 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class DesignBundlePipeline + include Pipeline + + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::DESIGN_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless portable.lfs_enabled? + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.design_repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index 1754f27137c..d5886d7bae7 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -10,16 +10,9 @@ module BulkImports relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION - transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer - - def extract(_context) - download_service.execute - decompression_service.execute - - project_attributes = json_decode(json_attributes) + extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation - BulkImports::Pipeline::ExtractedData.new(data: project_attributes) - end + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer def transform(_context, data) subrelations = config.portable_relations_tree.keys.map(&:to_s) @@ -39,51 +32,14 @@ module BulkImports end def after_run(_context) - FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) - end - - def json_attributes - @json_attributes ||= File.read(File.join(tmpdir, filename)) + extractor.remove_tmpdir end private - def tmpdir - @tmpdir ||= Dir.mktmpdir('bulk_imports') - end - def config @config ||= BulkImports::FileTransfer.config_for(portable) end - - def download_service - @download_service ||= BulkImports::FileDownloadService.new( - configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(self.class.relation), - tmpdir: tmpdir, - filename: compressed_filename - ) - end - - def decompression_service - @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: compressed_filename) - end - - def compressed_filename - "#{filename}.gz" - end - - def filename - "#{self.class.relation}.json" - end - - def json_decode(string) - Gitlab::Json.parse(string) - rescue JSON::ParserError => e - Gitlab::ErrorTracking.log_exception(e) - - raise BulkImports::Error, 'Incorrect JSON format' - end end end end diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb index 8f9c6a5749f..c77e53b9aec 100644 --- a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb @@ -9,6 +9,22 @@ module BulkImports relation_name 'releases' extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + + def after_run(_context) + super + + portable.releases.find_each do |release| + create_release_evidence(release) + end + end + + private + + def create_release_evidence(release) + return if release.historical_release? || release.upcoming_release? + + ::Releases::CreateEvidenceWorker.perform_async(release.id) + end end end end diff --git a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb new file mode 100644 index 00000000000..9a3c582642f --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class RepositoryBundlePipeline + include Pipeline + + abort_on_failure! + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::REPOSITORY_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end |