diff options
Diffstat (limited to 'lib/bulk_imports')
| -rw-r--r-- | lib/bulk_imports/groups/stage.rb | 18 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb | 71 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb | 50 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/releases_pipeline.rb | 16 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb | 71 | ||||
| -rw-r--r-- | lib/bulk_imports/projects/stage.rb | 26 | ||||
| -rw-r--r-- | lib/bulk_imports/stage.rb | 3 |
7 files changed, 204 insertions, 51 deletions
diff --git a/lib/bulk_imports/groups/stage.rb b/lib/bulk_imports/groups/stage.rb index c4db53424fd..0378a9c605d 100644 --- a/lib/bulk_imports/groups/stage.rb +++ b/lib/bulk_imports/groups/stage.rb @@ -5,6 +5,21 @@ module BulkImports class Stage < ::BulkImports::Stage private + # To skip the execution of a pipeline in a specific source instance version, define the attributes + # `minimum_source_version` and `maximum_source_version`. + # + # Use the `minimum_source_version` to inform that the pipeline needs to run when importing from source instances + # version greater than or equal to the specified minimum source version. For example, if the + # `minimum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.0, 15.1.1, 15.2.0, 16.0.0, etc. And it won't be executed when the source instance version + # is 15.0.1, 15.0.0, 14.10.0, etc. + # + # Use the `maximum_source_version` to inform that the pipeline needs to run when importing from source instance + # versions less than or equal to the specified maximum source version. For example, if the + # `maximum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.1 (patch), 15.1.0, 15.0.1, 15.0.0, 14.10.0, etc. And it won't be executed when the source + # instance version is 15.2.0, 15.2.1, 16.0.0, etc. + def config @config ||= { group: { @@ -21,7 +36,8 @@ module BulkImports }, namespace_settings: { pipeline: BulkImports::Groups::Pipelines::NamespaceSettingsPipeline, - stage: 1 + stage: 1, + minimum_source_version: '15.0.0' }, members: { pipeline: BulkImports::Common::Pipelines::MembersPipeline, diff --git a/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb new file mode 100644 index 00000000000..2d5231b0541 --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/design_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class DesignBundlePipeline + include Pipeline + + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::DESIGN_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless portable.lfs_enabled? + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.design_repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb index 1754f27137c..d5886d7bae7 100644 --- a/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/project_attributes_pipeline.rb @@ -10,16 +10,9 @@ module BulkImports relation_name BulkImports::FileTransfer::BaseConfig::SELF_RELATION - transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer - - def extract(_context) - download_service.execute - decompression_service.execute - - project_attributes = json_decode(json_attributes) + extractor ::BulkImports::Common::Extractors::JsonExtractor, relation: relation - BulkImports::Pipeline::ExtractedData.new(data: project_attributes) - end + transformer ::BulkImports::Common::Transformers::ProhibitedAttributesTransformer def transform(_context, data) subrelations = config.portable_relations_tree.keys.map(&:to_s) @@ -39,51 +32,14 @@ module BulkImports end def after_run(_context) - FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) - end - - def json_attributes - @json_attributes ||= File.read(File.join(tmpdir, filename)) + extractor.remove_tmpdir end private - def tmpdir - @tmpdir ||= Dir.mktmpdir('bulk_imports') - end - def config @config ||= BulkImports::FileTransfer.config_for(portable) end - - def download_service - @download_service ||= BulkImports::FileDownloadService.new( - configuration: context.configuration, - relative_url: context.entity.relation_download_url_path(self.class.relation), - tmpdir: tmpdir, - filename: compressed_filename - ) - end - - def decompression_service - @decompression_service ||= BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: compressed_filename) - end - - def compressed_filename - "#{filename}.gz" - end - - def filename - "#{self.class.relation}.json" - end - - def json_decode(string) - Gitlab::Json.parse(string) - rescue JSON::ParserError => e - Gitlab::ErrorTracking.log_exception(e) - - raise BulkImports::Error, 'Incorrect JSON format' - end end end end diff --git a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb index 8f9c6a5749f..c77e53b9aec 100644 --- a/lib/bulk_imports/projects/pipelines/releases_pipeline.rb +++ b/lib/bulk_imports/projects/pipelines/releases_pipeline.rb @@ -9,6 +9,22 @@ module BulkImports relation_name 'releases' extractor ::BulkImports::Common::Extractors::NdjsonExtractor, relation: relation + + def after_run(_context) + super + + portable.releases.find_each do |release| + create_release_evidence(release) + end + end + + private + + def create_release_evidence(release) + return if release.historical_release? || release.upcoming_release? + + ::Releases::CreateEvidenceWorker.perform_async(release.id) + end end end end diff --git a/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb new file mode 100644 index 00000000000..9a3c582642f --- /dev/null +++ b/lib/bulk_imports/projects/pipelines/repository_bundle_pipeline.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module BulkImports + module Projects + module Pipelines + class RepositoryBundlePipeline + include Pipeline + + abort_on_failure! + file_extraction_pipeline! + relation_name BulkImports::FileTransfer::ProjectConfig::REPOSITORY_BUNDLE_RELATION + + def extract(_context) + download_service.execute + decompression_service.execute + extraction_service.execute + + bundle_path = File.join(tmpdir, "#{self.class.relation}.bundle") + + BulkImports::Pipeline::ExtractedData.new(data: bundle_path) + end + + def load(_context, bundle_path) + Gitlab::Utils.check_path_traversal!(bundle_path) + Gitlab::Utils.check_allowed_absolute_path!(bundle_path, [Dir.tmpdir]) + + return unless File.exist?(bundle_path) + return if File.directory?(bundle_path) + return if File.lstat(bundle_path).symlink? + + portable.repository.create_from_bundle(bundle_path) + end + + def after_run(_) + FileUtils.remove_entry(tmpdir) if Dir.exist?(tmpdir) + end + + private + + def tar_filename + "#{self.class.relation}.tar" + end + + def targz_filename + "#{tar_filename}.gz" + end + + def download_service + BulkImports::FileDownloadService.new( + configuration: context.configuration, + relative_url: context.entity.relation_download_url_path(self.class.relation), + tmpdir: tmpdir, + filename: targz_filename + ) + end + + def decompression_service + BulkImports::FileDecompressionService.new(tmpdir: tmpdir, filename: targz_filename) + end + + def extraction_service + BulkImports::ArchiveExtractionService.new(tmpdir: tmpdir, filename: tar_filename) + end + + def tmpdir + @tmpdir ||= Dir.mktmpdir('bulk_imports') + end + end + end + end +end diff --git a/lib/bulk_imports/projects/stage.rb b/lib/bulk_imports/projects/stage.rb index 229df9c410d..acfa9163eae 100644 --- a/lib/bulk_imports/projects/stage.rb +++ b/lib/bulk_imports/projects/stage.rb @@ -5,6 +5,21 @@ module BulkImports class Stage < ::BulkImports::Stage private + # To skip the execution of a pipeline in a specific source instance version, define the attributes + # `minimum_source_version` and `maximum_source_version`. + # + # Use the `minimum_source_version` to inform that the pipeline needs to run when importing from source instances + # version greater than or equal to the specified minimum source version. For example, if the + # `minimum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.0, 15.1.1, 15.2.0, 16.0.0, etc. And it won't be executed when the source instance version + # is 15.0.1, 15.0.0, 14.10.0, etc. + # + # Use the `maximum_source_version` to inform that the pipeline needs to run when importing from source instance + # versions less than or equal to the specified maximum source version. For example, if the + # `maximum_source_version` is equal to 15.1.0, the pipeline will be executed when importing from source instances + # running versions 15.1.1 (patch), 15.1.0, 15.0.1, 15.0.0, 14.10.0, etc. And it won't be executed when the source + # instance version is 15.2.0, 15.2.1, 16.0.0, etc. + def config @config ||= { project: { @@ -13,6 +28,12 @@ module BulkImports }, repository: { pipeline: BulkImports::Projects::Pipelines::RepositoryPipeline, + maximum_source_version: '15.0.0', + stage: 1 + }, + repository_bundle: { + pipeline: BulkImports::Projects::Pipelines::RepositoryBundlePipeline, + minimum_source_version: '15.1.0', stage: 1 }, project_attributes: { @@ -95,6 +116,11 @@ module BulkImports pipeline: BulkImports::Common::Pipelines::LfsObjectsPipeline, stage: 5 }, + design: { + pipeline: BulkImports::Projects::Pipelines::DesignBundlePipeline, + minimum_source_version: '15.1.0', + stage: 5 + }, auto_devops: { pipeline: BulkImports::Projects::Pipelines::AutoDevopsPipeline, stage: 5 diff --git a/lib/bulk_imports/stage.rb b/lib/bulk_imports/stage.rb index 6cf394c5df0..b45ac139385 100644 --- a/lib/bulk_imports/stage.rb +++ b/lib/bulk_imports/stage.rb @@ -15,9 +15,6 @@ module BulkImports @pipelines ||= config .values .sort_by { |entry| entry[:stage] } - .map do |entry| - [entry[:stage], entry[:pipeline]] - end end private |