diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-12-20 17:22:11 +0300 |
commit | 0c872e02b2c822e3397515ec324051ff540f0cd5 (patch) | |
tree | ce2fb6ce7030e4dad0f4118d21ab6453e5938cdd /lib/gitlab/import_export | |
parent | f7e05a6853b12f02911494c4b3fe53d9540d74fc (diff) |
Add latest changes from gitlab-org/gitlab@15-7-stable-eev15.7.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
-rw-r--r-- | lib/gitlab/import_export/base/relation_factory.rb | 32 | ||||
-rw-r--r-- | lib/gitlab/import_export/decompressed_archive_size_validator.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/group/import_export.yml | 20 | ||||
-rw-r--r-- | lib/gitlab/import_export/json/legacy_reader.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/lfs_restorer.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/members_mapper.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/import_export.yml | 17 | ||||
-rw-r--r-- | lib/gitlab/import_export/project/tree_saver.rb | 16 | ||||
-rw-r--r-- | lib/gitlab/import_export/remote_stream_upload.rb | 2 | ||||
-rw-r--r-- | lib/gitlab/import_export/repo_restorer.rb | 5 |
10 files changed, 75 insertions, 39 deletions
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index b05d9cb2489..d1fd45882d3 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -211,23 +211,21 @@ module Gitlab def existing_or_new_object # Only find existing records to avoid mapping tables such as milestones # Otherwise always create the record, skipping the extra SELECT clause. - @existing_or_new_object ||= begin - if existing_object? - attribute_hash = attribute_hash_for(['events']) - - existing_object.assign_attributes(attribute_hash) if attribute_hash.any? - - existing_object - else - # Because of single-type inheritance, we need to be careful to use the `type` field - # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 - inheritance_column = relation_class.try(:inheritance_column) - inheritance_attributes = parsed_relation_hash.slice(inheritance_column) - object = relation_class.new(inheritance_attributes) - object.assign_attributes(parsed_relation_hash) - object - end - end + @existing_or_new_object ||= if existing_object? + attribute_hash = attribute_hash_for(['events']) + + existing_object.assign_attributes(attribute_hash) if attribute_hash.any? + + existing_object + else + # Because of single-type inheritance, we need to be careful to use the `type` field + # See https://gitlab.com/gitlab-org/gitlab/issues/34860#note_235321497 + inheritance_column = relation_class.try(:inheritance_column) + inheritance_attributes = parsed_relation_hash.slice(inheritance_column) + object = relation_class.new(inheritance_attributes) + object.assign_attributes(parsed_relation_hash) + object + end end def attribute_hash_for(attributes) diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index aa66fe8a5ae..564008e7a73 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -35,7 +35,7 @@ module Gitlab Timeout.timeout(TIMEOUT_LIMIT) do stderr_r, stderr_w = IO.pipe - stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w ) + stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) # When validation is performed on a small archive (e.g. 100 bytes) # `wait_thr` finishes before we can get process group id. Do not diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index a08efdf400b..7f3254be3e8 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -132,3 +132,23 @@ ee: - :push_event_payload - iterations_cadences: - :iterations + +# When associated resources are from outside the group, you might need to +# validate that a user who is exporting the group can access these +# associations. `include_if_exportable` accepts an array of associations for a +# resource. During export, the `exportable_association?` method on the +# resource is called with the association's name and user to validate if +# associated resource can be included in the export. +# +# This definition will call epic's `exportable_association?(:parent, +# current_user: current_user)` method and include epic's parent association +# for each epic only if the method returns true: +# +# include_if_exportable: +# group: +# epics: +# - :parent +include_if_exportable: + group: + epics: + - :parent diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb index dc80c92f507..ee360020556 100644 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ b/lib/gitlab/import_export/json/legacy_reader.rb @@ -27,7 +27,7 @@ module Gitlab end def read_hash - Gitlab::Json.parse(IO.read(@path)) + Gitlab::Json.parse(::File.read(@path)) rescue StandardError => e Gitlab::ErrorTracking.log_exception(e) raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb index 9931b09e9ca..83aab6d031e 100644 --- a/lib/gitlab/import_export/lfs_restorer.rb +++ b/lib/gitlab/import_export/lfs_restorer.rb @@ -71,7 +71,7 @@ module Gitlab @lfs_json ||= begin - json = IO.read(lfs_json_path) + json = File.read(lfs_json_path) Gitlab::Json.parse(json) rescue StandardError raise Gitlab::ImportExport::Error, 'Incorrect JSON format' diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index c94549a2b3f..0ad19f82e71 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -15,15 +15,13 @@ module Gitlab def map @map ||= - begin - @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| - if member['user'] - old_user_id = member['user']['id'] - existing_user_id = existing_users_email_map[get_email(member)] - hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) - else - add_team_member(member) - end + @exported_members.each_with_object(missing_keys_tracking_hash) do |member, hash| + if member['user'] + old_user_id = member['user']['id'] + existing_user_id = existing_users_email_map[get_email(member)] + hash[old_user_id] = existing_user_id if existing_user_id && add_team_member(member, existing_user_id) + else + add_team_member(member) end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 2d9c8d1108e..cc69ed55744 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -763,6 +763,19 @@ excluded_attributes: - :import_type - :import_source - :integrations + - :push_hooks_integrations + - :tag_push_hooks_integrations + - :issue_hooks_integrations + - :confidential_issue_hooks_integrations + - :merge_request_hooks_integrations + - :note_hooks_integrations + - :confidential_note_hooks_integrations + - :job_hooks_integrations + - :archive_trace_hooks_integrations + - :pipeline_hooks_integrations + - :wiki_page_hooks_integrations + - :deployment_hooks_integrations + - :alert_hooks_integrations - :mirror - :runners_token - :runners_token_encrypted @@ -1209,7 +1222,9 @@ ee: - :description iterations_cadence: - :title - + excluded_attributes: + project: + - :vulnerability_hooks_integrations preloads: issues: epic: diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index bd34cd3ff6e..05b96f7e8ce 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -81,15 +81,13 @@ module Gitlab end def json_writer - @json_writer ||= begin - if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) - full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) - else - full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') - end - end + @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) + else + full_path = File.join(@shared.export_path, ImportExport.project_filename) + Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') + end end end end diff --git a/lib/gitlab/import_export/remote_stream_upload.rb b/lib/gitlab/import_export/remote_stream_upload.rb index f3bd241c0bd..1fb3faf0767 100644 --- a/lib/gitlab/import_export/remote_stream_upload.rb +++ b/lib/gitlab/import_export/remote_stream_upload.rb @@ -25,6 +25,7 @@ module Gitlab end end end + class StreamError < StandardError attr_reader :response_body @@ -33,6 +34,7 @@ module Gitlab @response_body = response_body end end + class ChunkStream DEFAULT_BUFFER_SIZE = 128.kilobytes diff --git a/lib/gitlab/import_export/repo_restorer.rb b/lib/gitlab/import_export/repo_restorer.rb index 1c6629cf942..cc214d730fe 100644 --- a/lib/gitlab/import_export/repo_restorer.rb +++ b/lib/gitlab/import_export/repo_restorer.rb @@ -46,6 +46,11 @@ module Gitlab ) Repositories::DestroyService.new(repository).execute + + # Because Gitlab::Git::Repository#remove happens inside a run_after_commit + # callback in the Repositories::DestroyService#execute we need to trigger + # the callback. + repository.project.touch end end end |