diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
commit | 7021455bd1ed7b125c55eb1b33c5a01f2bc55ee0 (patch) | |
tree | 5bdc2229f5198d516781f8d24eace62fc7e589e9 /lib/gitlab/import_export | |
parent | 185b095e93520f96e9cfc31d9c3e69b498cdab7c (diff) |
Add latest changes from gitlab-org/gitlab@15-6-stable-eev15.6.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
7 files changed, 142 insertions, 11 deletions
diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index f6f65f85599..8c7a6c13246 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -85,11 +85,11 @@ module Gitlab while stack.any? model_name, relations = stack.pop - if relations.is_a?(Hash) - add_permitted_attributes(model_name, relations.keys) + next unless relations.is_a?(Hash) - stack.concat(relations.to_a) - end + add_permitted_attributes(model_name, relations.keys) + + stack.concat(relations.to_a) end @permitted_attributes diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 3c473449ec0..ed3858d0bf4 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -81,11 +81,11 @@ module Gitlab subrelation = relation_object.public_send(definition) association = relation_object.class.reflect_on_association(definition) - if association&.collection? && subrelation.size > MIN_RECORDS_SIZE - collection_subrelations[definition] = subrelation.records + next unless association&.collection? && subrelation.size > MIN_RECORDS_SIZE - subrelation.clear - end + collection_subrelations[definition] = subrelation.records + + subrelation.clear end end end diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index c98dcf7b848..aa66fe8a5ae 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -87,7 +87,6 @@ module Gitlab def validate_archive_path Gitlab::Utils.check_path_traversal!(@archive_path) - raise(ServiceError, 'Archive path is not a string') unless @archive_path.is_a?(String) raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path) end diff --git a/lib/gitlab/import_export/project/exported_relations_merger.rb b/lib/gitlab/import_export/project/exported_relations_merger.rb new file mode 100644 index 00000000000..dda3d00d608 --- /dev/null +++ b/lib/gitlab/import_export/project/exported_relations_merger.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ExportedRelationsMerger + include Gitlab::ImportExport::CommandLineUtil + + def initialize(export_job:, shared:) + @export_job = export_job + @shared = shared + end + + def save + Dir.mktmpdir do |dirpath| + export_job.relation_exports.each do |relation_export| + relation = relation_export.relation + upload = relation_export.upload + filename = upload.export_file.filename + + tar_gz_full_path = File.join(dirpath, filename) + decompress_path = File.join(dirpath, relation) + Gitlab::Utils.check_path_traversal!(tar_gz_full_path) + Gitlab::Utils.check_path_traversal!(decompress_path) + + # Download tar.gz + download_or_copy_upload( + upload.export_file, tar_gz_full_path, size_limit: relation_export.upload.export_file.size + ) + + # Decompress tar.gz + mkdir_p(decompress_path) + untar_zxf(dir: decompress_path, archive: tar_gz_full_path) + File.delete(tar_gz_full_path) + + # Merge decompressed files into export_path + RecursiveMergeFolders.merge(decompress_path, shared.export_path) + FileUtils.rm_r(decompress_path) + rescue StandardError => e + shared.error(e) + false + end + end + + shared.errors.empty? + end + + private + + attr_reader :shared, :export_job + + delegate :project, to: :export_job + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index fb44aaf094e..2d9c8d1108e 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -302,6 +302,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level prometheus_metrics: - :created_at - :updated_at @@ -585,7 +586,7 @@ included_attributes: - :target_sha pipeline_metadata: - :project_id - - :title + - :name stages: - :name - :status @@ -717,6 +718,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index 8e91adac196..967239e17c1 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,7 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project', + exportable_path: 'tree/project', current_user: nil ) end diff --git a/lib/gitlab/import_export/recursive_merge_folders.rb b/lib/gitlab/import_export/recursive_merge_folders.rb new file mode 100644 index 00000000000..982358699bd --- /dev/null +++ b/lib/gitlab/import_export/recursive_merge_folders.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +# +# This class is used by Import/Export to move files and folders from a source folders into a target folders +# that can already have the same folders in it, resolving in a merged folder. +# +# Example: +# +# source path +# |-- tree +# | |-- project +# | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | |-- folder2 +# | | |-- image2.png +# +# target path +# |-- tree +# | |-- project +# | |-- issues.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image3.png +# | |-- folder3 +# | | |-- image4.png +# +# target path after merge +# |-- tree +# | |-- project +# | | |-- issues.ndjson +# | | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | | |-- image3.png +# | |-- folder2 +# | | |-- image2.png +# | |-- folder3 +# | | |-- image4.png + +module Gitlab + module ImportExport + class RecursiveMergeFolders + DEFAULT_DIR_MODE = 0o700 + + def self.merge(source_path, target_path) + Gitlab::Utils.check_path_traversal!(source_path) + Gitlab::Utils.check_path_traversal!(target_path) + Gitlab::Utils.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) + + recursive_merge(source_path, target_path) + end + + def self.recursive_merge(source_path, target_path) + Dir.children(source_path).each do |child| + source_child = File.join(source_path, child) + target_child = File.join(target_path, child) + + next if File.lstat(source_child).symlink? + + if File.directory?(source_child) + FileUtils.mkdir_p(target_child, mode: DEFAULT_DIR_MODE) unless File.exist?(target_child) + recursive_merge(source_child, target_child) + else + FileUtils.mv(source_child, target_child) + end + end + end + + private_class_method :recursive_merge + end + end +end |