diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-09-20 02:18:09 +0300 |
commit | 6ed4ec3e0b1340f96b7c043ef51d1b33bbe85fde (patch) | |
tree | dc4d20fe6064752c0bd323187252c77e0a89144b /lib/gitlab/import_export | |
parent | 9868dae7fc0655bd7ce4a6887d4e6d487690eeed (diff) |
Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42
Diffstat (limited to 'lib/gitlab/import_export')
18 files changed, 279 insertions, 53 deletions
diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 4abc3da1190..8843b4f5755 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -12,6 +12,7 @@ module Gitlab @methods = config[:methods] || {} @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} + @include_if_exportable = config[:include_if_exportable] || {} end def find_root(model_key) @@ -35,7 +36,8 @@ module Gitlab methods: @methods[model_key], include: resolve_model_tree(model_tree), preload: resolve_preloads(model_key, model_tree), - export_reorder: @export_reorders[model_key] + export_reorder: @export_reorders[model_key], + include_if_exportable: @include_if_exportable[model_key] }.compact end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 1cbfcbdb595..bbec473d29d 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -31,6 +31,8 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze + attr_reader :relation_name, :importable + def self.create(*args, **kwargs) new(*args, **kwargs).create end diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index ea989487ebd..3c473449ec0 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -58,8 +58,19 @@ module Gitlab records.each_slice(BATCH_SIZE) do |batch| valid_records, invalid_records = batch.partition { |record| record.valid? } - invalid_subrelations << invalid_records relation_object.public_send(relation_name) << valid_records + + # Attempt to save some of the invalid subrelations, as they might be valid after all. + # For example, a merge request `Approval` validates presence of merge_request_id. + # It is not present at a time of calling `#valid?` above, since it's indeed missing. + # However, when saving such subrelation against already persisted merge request + # such validation won't fail (e.g. `merge_request.approvals << Approval.new(user_id: 1)`), + # as we're operating on a merge request that has `id` present. + invalid_records.each do |invalid_record| + relation_object.public_send(relation_name) << invalid_record + + invalid_subrelations << invalid_record unless invalid_record.persisted? + end end end end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 8df5d52bf77..a08efdf400b 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -27,6 +27,26 @@ included_attributes: - :name namespace_settings: - :prevent_sharing_groups_outside_hierarchy + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :title + - :description + - :sequence + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -44,6 +64,23 @@ excluded_attributes: - :max_pages_size epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + iterations: *iteration_excluded_definition methods: labels: @@ -83,6 +120,7 @@ ee: - events: - :push_event_payload - :system_note_metadata + - :resource_state_events - boards: - :board_assignee - :milestone @@ -92,3 +130,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 082d2346f3d..6507def7d01 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -24,6 +24,29 @@ included_attributes: - :username author: - :name + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :active + - :automatic + - :roll_over + - :title + - :description + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -41,6 +64,18 @@ excluded_attributes: - :extra_shared_runners_minutes_limit epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + iterations: *iteration_excluded_definition methods: labels: @@ -79,6 +114,7 @@ ee: - :award_emoji - events: - :push_event_payload + - :resource_state_events - boards: - :board_assignee - :milestone @@ -88,3 +124,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 8b39362b6bb..fa9e765b33a 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -68,23 +68,23 @@ module Gitlab def restorer @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - relation_reader: relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader, - importable: @group, + user: @user, + shared: @shared, + relation_reader: relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader, + importable: @group, importable_attributes: @group_attributes, - importable_path: nil + importable_path: nil ) end def create_group(group_hash:, parent_group:) group_params = { - name: group_hash['name'], - path: group_hash['path'], + name: group_hash['name'], + path: group_hash['path'], parent_id: parent_group&.id, visibility_level: sub_group_visibility_level(group_hash, parent_group) } diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 258078d595b..1b8436c4ed9 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -5,10 +5,11 @@ module Gitlab module Group class RelationFactory < Base::RelationFactory OVERRIDES = { - labels: :group_labels, + labels: :group_labels, priorities: :label_priorities, - label: :group_label, - parent: :epic + label: :group_label, + parent: :epic, + iterations_cadences: 'Iterations::Cadence' }.freeze EXISTING_OBJECT_RELATIONS = %i[ @@ -25,7 +26,10 @@ module Gitlab private def setup_models - setup_note if @relation_name == :notes + case @relation_name + when :notes then setup_note + when :'Iterations::Cadence' then setup_iterations_cadence + end update_group_references end @@ -44,6 +48,10 @@ module Gitlab def use_attributes_permitter? false end + + def setup_iterations_cadence + @relation_hash['automatic'] = false + end end end end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index fab677bd772..5a78f2fb531 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -136,9 +136,9 @@ module Gitlab attributes_permitter.permit(importable_class_sym, params) else Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, + relation_hash: params, relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) + excluded_keys: excluded_keys_for_relation(importable_class_sym)) end end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 796b9258e57..b4c86c3fc7f 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -46,7 +46,8 @@ module Gitlab group, group_tree, json_writer, - exportable_path: "groups/#{group.id}" + exportable_path: "groups/#{group.id}", + current_user: @current_user ).execute end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 78f43f79072..99396d64779 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -6,11 +6,7 @@ module Gitlab class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil - BATCH_SIZE = 2 - - def self.batch_size(exportable) - BATCH_SIZE - end + BATCH_SIZE = 100 class Raw < String def to_json(*_args) @@ -18,8 +14,9 @@ module Gitlab end end - def initialize(exportable, relations_schema, json_writer, exportable_path:, logger: Gitlab::Export::Logger) + def initialize(exportable, relations_schema, json_writer, current_user:, exportable_path:, logger: Gitlab::Export::Logger) @exportable = exportable + @current_user = current_user @exportable_path = exportable_path @relations_schema = relations_schema @json_writer = json_writer @@ -63,7 +60,7 @@ module Gitlab private - attr_reader :json_writer, :relations_schema, :exportable, :logger + attr_reader :json_writer, :relations_schema, :exportable, :logger, :current_user def serialize_many_relations(key, records, options) log_relation_export(key, records.size) @@ -77,7 +74,7 @@ module Gitlab batch.each do |record| before_read_callback(record) - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) after_read_callback(record) end @@ -87,8 +84,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def exportable_json_record(record, options, key) + associations = relations_schema[:include_if_exportable]&.dig(key) + return Raw.new(record.to_json(options)) unless associations && options[:include] + + filtered_options = options.deep_dup + associations.each do |association| + filtered_options[:include].delete_if do |option| + !exportable_json_association?(option, record, association.to_sym) + end + end + + Raw.new(record.to_json(filtered_options)) + end + + def exportable_json_association?(option, record, association) + return true unless option.has_key?(association) + return false unless record.respond_to?(:exportable_association?) + + record.exportable_association?(association, current_user: current_user) + end + def batch(relation, key) - opts = { of: batch_size } + opts = { of: BATCH_SIZE } order_by = reorders(relation, key) # we need to sort issues by non primary key column(relative_position) @@ -115,7 +133,7 @@ module Gitlab enumerator = Enumerator.new do |items| records.each do |record| - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) end end @@ -125,7 +143,7 @@ module Gitlab def serialize_single_relation(key, record, options) log_relation_export(key) - json = Raw.new(record.to_json(options)) + json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) end @@ -138,10 +156,6 @@ module Gitlab relations_schema[:preload] end - def batch_size - @batch_size ||= self.class.batch_size(@exportable) - end - def reorders(relation, key) export_reorder = relations_schema[:export_reorder]&.dig(key) return unless export_reorder diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index c6b961ea210..cf75a2c7fa8 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -7,7 +7,7 @@ module Gitlab def serialize(exportable, relations_tree) Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree, batch_size: batch_size(exportable)) + .new(exportable, relations_tree) .execute end @@ -18,12 +18,6 @@ module Gitlab File.write(File.join(dir_path, filename), tree_json) end - - private - - def batch_size(exportable) - Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) - end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b1f2a17d4b7..c94549a2b3f 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -139,7 +139,7 @@ module Gitlab end def parsed_hash(member) - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, relation_class: relation_class) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index c5b8f3fd35b..33e4823f192 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -29,6 +29,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - designs: - notes: - :author @@ -82,6 +85,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - :external_pull_requests - ci_pipelines: - notes: @@ -287,6 +293,7 @@ included_attributes: - :forking_access_level - :metrics_dashboard_access_level - :operations_access_level + - :monitor_access_level - :analytics_access_level - :security_and_compliance_access_level - :container_registry_access_level @@ -551,6 +558,7 @@ included_attributes: - :failure_reason - :scheduled_at - :scheduling_type + - :ci_stage ci_pipelines: - :ref - :sha @@ -599,7 +607,6 @@ included_attributes: merge_request_assignees: - :user_id - :created_at - - :state merge_request_reviewers: - :user_id - :created_at @@ -699,6 +706,7 @@ included_attributes: - :metrics_dashboard_access_level - :analytics_access_level - :operations_access_level + - :monitor_access_level - :security_and_compliance_access_level - :container_registry_access_level - :package_registry_access_level @@ -721,6 +729,18 @@ included_attributes: - :build_git_strategy - :build_enabled - :security_and_compliance_enabled + resource_milestone_events: + - :user_id + - :action + - :created_at + - :state + resource_state_events: + - :user_id + - :state + - :created_at + - :source_commit + - :close_after_error_tracking_resolve + - :close_auto_resolve_prometheus_alert # Do not include the following attributes for the models specified. excluded_attributes: @@ -989,6 +1009,46 @@ excluded_attributes: milestone_releases: - :milestone_id - :release_id + resource_milestone_events: + - :id + - :issue_id + - :merge_request_id + - :milestone_id + resource_state_events: + - :id + - :issue_id + - :merge_request_id + - :epic_id + - :source_merge_request_id + iteration: + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + resource_iteration_events: + - :id + - :issue_id + - :merge_request_id + - :iteration_id + iterations_cadence: + - :id + - :last_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :description + - :sequence + methods: notes: - :type @@ -1062,6 +1122,11 @@ ee: - epic_issue: - :epic - :issuable_sla + - iteration: + - :iterations_cadence + - resource_iteration_events: + - iteration: + - :iterations_cadence - protected_branches: - :unprotect_access_levels - protected_environments: @@ -1120,5 +1185,44 @@ ee: - :auto_fix_dependency_scanning - :auto_fix_sast project: - - :requirements_enabled - - :requirements_access_level + - :requirements_enabled + - :requirements_access_level + resource_iteration_events: + - :user_id + - :action + - :created_at + iteration: + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations_cadence: + - :title + + preloads: + issues: + epic: + + # When associated resources are from outside the project, you might need to + # validate that a user who is exporting the project or group can access these + # associations. `include_if_exportable` accepts an array of associations for a + # resource. During export, the `exportable_association?` method on the + # resource is called with the association's name and user to validate if + # associated resource can be included in the export. + # + # This definition will call issue's `exportable_association?(:epic_issue, + # current_user: current_user)` method and include issue's epic_issue association + # for each issue only if the method returns true: + # + # Example: + # include_if_exportable: + # project: + # issues: + # - epic_issue + include_if_exportable: + project: + issues: + - :epic_issue diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 59bb8af750e..89f2b36ea58 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -80,8 +80,8 @@ module Gitlab def import_params { namespace_id: namespace.id, - path: project_path, - file: File.open(file_path) + path: project_path, + file: File.open(file_path) } end diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index bf60d115a25..50a67a746f8 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -21,7 +21,7 @@ module Gitlab end def find - return if epic? && group.nil? + return if group_relation_without_group? return find_diff_commit_user if diff_commit_user? return find_diff_commit if diff_commit? @@ -60,7 +60,7 @@ module Gitlab def prepare_attributes attributes.dup.tap do |atts| - atts.delete('group') unless epic? + atts.delete('group') unless epic? || iteration? if label? atts['type'] = 'ProjectLabel' # Always create project labels @@ -141,6 +141,10 @@ module Gitlab klass == MergeRequestDiffCommit end + def iteration? + klass == Iteration + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -157,7 +161,13 @@ module Gitlab milestone.ensure_project_iid! milestone.save! end + + def group_relation_without_group? + (epic? || iteration?) && group.nil? + end end end end end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_mod diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index b40827e36f8..8e91adac196 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,8 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project' + exportable_path: 'project', + current_user: nil ) end diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb index 6e9548f393a..47196db6f8a 100644 --- a/lib/gitlab/import_export/project/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -5,7 +5,7 @@ module Gitlab module Project class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze private diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1b54e4b975e..bd34cd3ff6e 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -50,7 +50,8 @@ module Gitlab reader.project_tree, json_writer, exportable_path: "project", - logger: @logger + logger: @logger, + current_user: @current_user ) Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do |