Add latest changes from gitlab-org/gitlab@15-4-stable-eev15.4.0-rc42

18 files changed, 279 insertions, 53 deletions

diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb
index 4abc3da1190..8843b4f5755 100644
--- a/lib/gitlab/import_export/attributes_finder.rb
+++ b/lib/gitlab/import_export/attributes_finder.rb
@@ -12,6 +12,7 @@ module Gitlab
         @methods = config[:methods] || {}
         @preloads = config[:preloads] || {}
         @export_reorders = config[:export_reorders] || {}
+        @include_if_exportable = config[:include_if_exportable] || {}
       end
 
       def find_root(model_key)
@@ -35,7 +36,8 @@ module Gitlab
           methods: @methods[model_key],
           include: resolve_model_tree(model_tree),
           preload: resolve_preloads(model_key, model_tree),
-          export_reorder: @export_reorders[model_key]
+          export_reorder: @export_reorders[model_key],
+          include_if_exportable: @include_if_exportable[model_key]
         }.compact
       end
 
diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb
index 1cbfcbdb595..bbec473d29d 100644
--- a/lib/gitlab/import_export/base/relation_factory.rb
+++ b/lib/gitlab/import_export/base/relation_factory.rb
@@ -31,6 +31,8 @@ module Gitlab
 
         TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze
 
+        attr_reader :relation_name, :importable
+
         def self.create(*args, **kwargs)
           new(*args, **kwargs).create
         end
diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb
index ea989487ebd..3c473449ec0 100644
--- a/lib/gitlab/import_export/base/relation_object_saver.rb
+++ b/lib/gitlab/import_export/base/relation_object_saver.rb
@@ -58,8 +58,19 @@ module Gitlab
             records.each_slice(BATCH_SIZE) do |batch|
               valid_records, invalid_records = batch.partition { |record| record.valid? }
 
-              invalid_subrelations << invalid_records
               relation_object.public_send(relation_name) << valid_records
+
+              # Attempt to save some of the invalid subrelations, as they might be valid after all.
+              # For example, a merge request `Approval` validates presence of merge_request_id.
+              # It is not present at a time of calling `#valid?` above, since it's indeed missing.
+              # However, when saving such subrelation against already persisted merge request
+              # such validation won't fail (e.g. `merge_request.approvals << Approval.new(user_id: 1)`),
+              # as we're operating on a merge request that has `id` present.
+              invalid_records.each do |invalid_record|
+                relation_object.public_send(relation_name) << invalid_record
+
+                invalid_subrelations << invalid_record unless invalid_record.persisted?
+              end
             end
           end
         end
diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml
index 8df5d52bf77..a08efdf400b 100644
--- a/lib/gitlab/import_export/group/import_export.yml
+++ b/lib/gitlab/import_export/group/import_export.yml
@@ -27,6 +27,26 @@ included_attributes:
     - :name
   namespace_settings:
     - :prevent_sharing_groups_outside_hierarchy
+  iterations_cadence: &iterations_cadence_definition
+    - :group_id
+    - :created_at
+    - :updated_at
+    - :start_date
+    - :active
+    - :roll_over
+    - :title
+    - :description
+    - :sequence
+  iterations_cadences: *iterations_cadence_definition
+  iteration: &iteration_definition
+    - :iid
+    - :created_at
+    - :updated_at
+    - :start_date
+    - :due_date
+    - :group_id
+    - :description
+  iterations: *iteration_definition
 
 excluded_attributes:
   group:
@@ -44,6 +64,23 @@ excluded_attributes:
     - :max_pages_size
   epics:
     - :state_id
+  iterations_cadence: &iterations_cadence_definition
+    - :id
+    - :next_run_date
+    - :duration_in_weeks
+    - :iterations_in_advance
+    - :automatic
+  iterations_cadences: *iterations_cadence_definition
+  iteration: &iteration_excluded_definition
+    - :id
+    - :title
+    - :title_html
+    - :project_id
+    - :description_html
+    - :cached_markdown_version
+    - :iterations_cadence_id
+    - :sequence
+  iterations: *iteration_excluded_definition
 
 methods:
   labels:
@@ -83,6 +120,7 @@ ee:
           - events:
             - :push_event_payload
           - :system_note_metadata
+        - :resource_state_events
       - boards:
         - :board_assignee
         - :milestone
@@ -92,3 +130,5 @@ ee:
           - milestone:
             - events:
               - :push_event_payload
+      - iterations_cadences:
+        - :iterations
diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml
index 082d2346f3d..6507def7d01 100644
--- a/lib/gitlab/import_export/group/legacy_import_export.yml
+++ b/lib/gitlab/import_export/group/legacy_import_export.yml
@@ -24,6 +24,29 @@ included_attributes:
     - :username
   author:
     - :name
+  iterations_cadence: &iterations_cadence_definition
+    - :group_id
+    - :created_at
+    - :updated_at
+    - :start_date
+    - :next_run_date
+    - :duration_in_weeks
+    - :iterations_in_advance
+    - :active
+    - :automatic
+    - :roll_over
+    - :title
+    - :description
+  iterations_cadences: *iterations_cadence_definition
+  iteration: &iteration_definition
+    - :iid
+    - :created_at
+    - :updated_at
+    - :start_date
+    - :due_date
+    - :group_id
+    - :description
+  iterations: *iteration_definition
 
 excluded_attributes:
   group:
@@ -41,6 +64,18 @@ excluded_attributes:
     - :extra_shared_runners_minutes_limit
   epics:
     - :state_id
+  iterations_cadence: &iterations_cadence_definition
+    - :id
+  iterations_cadences: *iterations_cadence_definition
+  iteration: &iteration_excluded_definition
+    - :id
+    - :title
+    - :title_html
+    - :project_id
+    - :description_html
+    - :cached_markdown_version
+    - :iterations_cadence_id
+  iterations: *iteration_excluded_definition
 
 methods:
   labels:
@@ -79,6 +114,7 @@ ee:
           - :award_emoji
           - events:
             - :push_event_payload
+        - :resource_state_events
       - boards:
         - :board_assignee
         - :milestone
@@ -88,3 +124,5 @@ ee:
           - milestone:
             - events:
               - :push_event_payload
+      - iterations_cadences:
+          - :iterations
diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
index 8b39362b6bb..fa9e765b33a 100644
--- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb
@@ -68,23 +68,23 @@ module Gitlab
 
         def restorer
           @relation_tree_restorer ||= RelationTreeRestorer.new(
-            user:                  @user,
-            shared:                @shared,
-            relation_reader:       relation_reader,
-            members_mapper:        members_mapper,
-            object_builder:        object_builder,
-            relation_factory:      relation_factory,
-            reader:                reader,
-            importable:            @group,
+            user: @user,
+            shared: @shared,
+            relation_reader: relation_reader,
+            members_mapper: members_mapper,
+            object_builder: object_builder,
+            relation_factory: relation_factory,
+            reader: reader,
+            importable: @group,
             importable_attributes: @group_attributes,
-            importable_path:       nil
+            importable_path: nil
           )
         end
 
         def create_group(group_hash:, parent_group:)
           group_params = {
-            name:      group_hash['name'],
-            path:      group_hash['path'],
+            name: group_hash['name'],
+            path: group_hash['path'],
             parent_id: parent_group&.id,
             visibility_level: sub_group_visibility_level(group_hash, parent_group)
           }
diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb
index 258078d595b..1b8436c4ed9 100644
--- a/lib/gitlab/import_export/group/relation_factory.rb
+++ b/lib/gitlab/import_export/group/relation_factory.rb
@@ -5,10 +5,11 @@ module Gitlab
     module Group
       class RelationFactory < Base::RelationFactory
         OVERRIDES = {
-          labels:     :group_labels,
+          labels: :group_labels,
           priorities: :label_priorities,
-          label:      :group_label,
-          parent:     :epic
+          label: :group_label,
+          parent: :epic,
+          iterations_cadences: 'Iterations::Cadence'
         }.freeze
 
         EXISTING_OBJECT_RELATIONS = %i[
@@ -25,7 +26,10 @@ module Gitlab
         private
 
         def setup_models
-          setup_note if @relation_name == :notes
+          case @relation_name
+          when :notes then setup_note
+          when :'Iterations::Cadence' then setup_iterations_cadence
+          end
 
           update_group_references
         end
@@ -44,6 +48,10 @@ module Gitlab
         def use_attributes_permitter?
           false
         end
+
+        def setup_iterations_cadence
+          @relation_hash['automatic'] = false
+        end
       end
     end
   end
diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb
index fab677bd772..5a78f2fb531 100644
--- a/lib/gitlab/import_export/group/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb
@@ -136,9 +136,9 @@ module Gitlab
             attributes_permitter.permit(importable_class_sym, params)
           else
             Gitlab::ImportExport::AttributeCleaner.clean(
-              relation_hash:  params,
+              relation_hash: params,
               relation_class: importable_class,
-              excluded_keys:  excluded_keys_for_relation(importable_class_sym))
+              excluded_keys: excluded_keys_for_relation(importable_class_sym))
           end
         end
 
diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb
index 796b9258e57..b4c86c3fc7f 100644
--- a/lib/gitlab/import_export/group/tree_saver.rb
+++ b/lib/gitlab/import_export/group/tree_saver.rb
@@ -46,7 +46,8 @@ module Gitlab
             group,
             group_tree,
             json_writer,
-            exportable_path: "groups/#{group.id}"
+            exportable_path: "groups/#{group.id}",
+            current_user: @current_user
           ).execute
         end
 
diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb
index 78f43f79072..99396d64779 100644
--- a/lib/gitlab/import_export/json/streaming_serializer.rb
+++ b/lib/gitlab/import_export/json/streaming_serializer.rb
@@ -6,11 +6,7 @@ module Gitlab
       class StreamingSerializer
         include Gitlab::ImportExport::CommandLineUtil
 
-        BATCH_SIZE = 2
-
-        def self.batch_size(exportable)
-          BATCH_SIZE
-        end
+        BATCH_SIZE = 100
 
         class Raw < String
           def to_json(*_args)
@@ -18,8 +14,9 @@ module Gitlab
           end
         end
 
-        def initialize(exportable, relations_schema, json_writer, exportable_path:, logger: Gitlab::Export::Logger)
+        def initialize(exportable, relations_schema, json_writer, current_user:, exportable_path:, logger: Gitlab::Export::Logger)
           @exportable = exportable
+          @current_user = current_user
           @exportable_path = exportable_path
           @relations_schema = relations_schema
           @json_writer = json_writer
@@ -63,7 +60,7 @@ module Gitlab
 
         private
 
-        attr_reader :json_writer, :relations_schema, :exportable, :logger
+        attr_reader :json_writer, :relations_schema, :exportable, :logger, :current_user
 
         def serialize_many_relations(key, records, options)
           log_relation_export(key, records.size)
@@ -77,7 +74,7 @@ module Gitlab
               batch.each do |record|
                 before_read_callback(record)
 
-                items << Raw.new(record.to_json(options))
+                items << exportable_json_record(record, options, key)
 
                 after_read_callback(record)
               end
@@ -87,8 +84,29 @@ module Gitlab
           json_writer.write_relation_array(@exportable_path, key, enumerator)
         end
 
+        def exportable_json_record(record, options, key)
+          associations = relations_schema[:include_if_exportable]&.dig(key)
+          return Raw.new(record.to_json(options)) unless associations && options[:include]
+
+          filtered_options = options.deep_dup
+          associations.each do |association|
+            filtered_options[:include].delete_if do |option|
+              !exportable_json_association?(option, record, association.to_sym)
+            end
+          end
+
+          Raw.new(record.to_json(filtered_options))
+        end
+
+        def exportable_json_association?(option, record, association)
+          return true unless option.has_key?(association)
+          return false unless record.respond_to?(:exportable_association?)
+
+          record.exportable_association?(association, current_user: current_user)
+        end
+
         def batch(relation, key)
-          opts = { of: batch_size }
+          opts = { of: BATCH_SIZE }
           order_by = reorders(relation, key)
 
           # we need to sort issues by non primary key column(relative_position)
@@ -115,7 +133,7 @@ module Gitlab
 
           enumerator = Enumerator.new do |items|
             records.each do |record|
-              items << Raw.new(record.to_json(options))
+              items << exportable_json_record(record, options, key)
             end
           end
 
@@ -125,7 +143,7 @@ module Gitlab
         def serialize_single_relation(key, record, options)
           log_relation_export(key)
 
-          json = Raw.new(record.to_json(options))
+          json = exportable_json_record(record, options, key)
 
           json_writer.write_relation(@exportable_path, key, json)
         end
@@ -138,10 +156,6 @@ module Gitlab
           relations_schema[:preload]
         end
 
-        def batch_size
-          @batch_size ||= self.class.batch_size(@exportable)
-        end
-
         def reorders(relation, key)
           export_reorder = relations_schema[:export_reorder]&.dig(key)
           return unless export_reorder
diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
index c6b961ea210..cf75a2c7fa8 100644
--- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb
+++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb
@@ -7,7 +7,7 @@ module Gitlab
 
       def serialize(exportable, relations_tree)
         Gitlab::ImportExport::FastHashSerializer
-          .new(exportable, relations_tree, batch_size: batch_size(exportable))
+          .new(exportable, relations_tree)
           .execute
       end
 
@@ -18,12 +18,6 @@ module Gitlab
 
         File.write(File.join(dir_path, filename), tree_json)
       end
-
-      private
-
-      def batch_size(exportable)
-        Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable)
-      end
     end
   end
 end
diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb
index b1f2a17d4b7..c94549a2b3f 100644
--- a/lib/gitlab/import_export/members_mapper.rb
+++ b/lib/gitlab/import_export/members_mapper.rb
@@ -139,7 +139,7 @@ module Gitlab
       end
 
       def parsed_hash(member)
-        Gitlab::ImportExport::AttributeCleaner.clean(relation_hash:  member.deep_stringify_keys,
+        Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys,
                                                      relation_class: relation_class)
       end
 
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index c5b8f3fd35b..33e4823f192 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -29,6 +29,9 @@ tree:
       - resource_label_events:
         - label:
           - :priorities
+      - resource_milestone_events:
+        - :milestone
+      - :resource_state_events
       - designs:
         - notes:
           - :author
@@ -82,6 +85,9 @@ tree:
       - resource_label_events:
         - label:
           - :priorities
+      - resource_milestone_events:
+        - :milestone
+      - :resource_state_events
     - :external_pull_requests
     - ci_pipelines:
       - notes:
@@ -287,6 +293,7 @@ included_attributes:
     - :forking_access_level
     - :metrics_dashboard_access_level
     - :operations_access_level
+    - :monitor_access_level
     - :analytics_access_level
     - :security_and_compliance_access_level
     - :container_registry_access_level
@@ -551,6 +558,7 @@ included_attributes:
     - :failure_reason
     - :scheduled_at
     - :scheduling_type
+    - :ci_stage
   ci_pipelines:
     - :ref
     - :sha
@@ -599,7 +607,6 @@ included_attributes:
   merge_request_assignees:
     - :user_id
     - :created_at
-    - :state
   merge_request_reviewers:
     - :user_id
     - :created_at
@@ -699,6 +706,7 @@ included_attributes:
     - :metrics_dashboard_access_level
     - :analytics_access_level
     - :operations_access_level
+    - :monitor_access_level
     - :security_and_compliance_access_level
     - :container_registry_access_level
     - :package_registry_access_level
@@ -721,6 +729,18 @@ included_attributes:
     - :build_git_strategy
     - :build_enabled
     - :security_and_compliance_enabled
+  resource_milestone_events:
+    - :user_id
+    - :action
+    - :created_at
+    - :state
+  resource_state_events:
+    - :user_id
+    - :state
+    - :created_at
+    - :source_commit
+    - :close_after_error_tracking_resolve
+    - :close_auto_resolve_prometheus_alert
 
 # Do not include the following attributes for the models specified.
 excluded_attributes:
@@ -989,6 +1009,46 @@ excluded_attributes:
   milestone_releases:
     - :milestone_id
     - :release_id
+  resource_milestone_events:
+    - :id
+    - :issue_id
+    - :merge_request_id
+    - :milestone_id
+  resource_state_events:
+    - :id
+    - :issue_id
+    - :merge_request_id
+    - :epic_id
+    - :source_merge_request_id
+  iteration:
+    - :id
+    - :title
+    - :title_html
+    - :project_id
+    - :description_html
+    - :cached_markdown_version
+    - :iterations_cadence_id
+    - :sequence
+  resource_iteration_events:
+    - :id
+    - :issue_id
+    - :merge_request_id
+    - :iteration_id
+  iterations_cadence:
+    - :id
+    - :last_run_date
+    - :duration_in_weeks
+    - :iterations_in_advance
+    - :automatic
+    - :group_id
+    - :created_at
+    - :updated_at
+    - :start_date
+    - :active
+    - :roll_over
+    - :description
+    - :sequence
+
 methods:
   notes:
     - :type
@@ -1062,6 +1122,11 @@ ee:
         - epic_issue:
           - :epic
         - :issuable_sla
+        - iteration:
+          - :iterations_cadence
+        - resource_iteration_events:
+          - iteration:
+            - :iterations_cadence
       - protected_branches:
         - :unprotect_access_levels
       - protected_environments:
@@ -1120,5 +1185,44 @@ ee:
       - :auto_fix_dependency_scanning
       - :auto_fix_sast
     project:
-    - :requirements_enabled
-    - :requirements_access_level
+      - :requirements_enabled
+      - :requirements_access_level
+    resource_iteration_events:
+      - :user_id
+      - :action
+      - :created_at
+    iteration:
+      - :iid
+      - :created_at
+      - :updated_at
+      - :start_date
+      - :due_date
+      - :group_id
+      - :description
+    iterations_cadence:
+      - :title
+
+  preloads:
+    issues:
+      epic:
+
+  # When associated resources are from outside the project, you might need to
+  # validate that a user who is exporting the project or group can access these
+  # associations. `include_if_exportable` accepts an array of associations for a
+  # resource. During export, the `exportable_association?` method on the
+  # resource is called with the association's name and user to validate if
+  # associated resource can be included in the export.
+  #
+  # This definition will call issue's `exportable_association?(:epic_issue,
+  # current_user: current_user)` method and include issue's epic_issue association
+  # for each issue only if the method returns true:
+  #
+  # Example:
+  # include_if_exportable:
+  #   project:
+  #     issues:
+  #       - epic_issue
+  include_if_exportable:
+    project:
+      issues:
+        - :epic_issue
diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb
index 59bb8af750e..89f2b36ea58 100644
--- a/lib/gitlab/import_export/project/import_task.rb
+++ b/lib/gitlab/import_export/project/import_task.rb
@@ -80,8 +80,8 @@ module Gitlab
         def import_params
           {
             namespace_id: namespace.id,
-            path:         project_path,
-            file:         File.open(file_path)
+            path: project_path,
+            file: File.open(file_path)
           }
         end
 
diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb
index bf60d115a25..50a67a746f8 100644
--- a/lib/gitlab/import_export/project/object_builder.rb
+++ b/lib/gitlab/import_export/project/object_builder.rb
@@ -21,7 +21,7 @@ module Gitlab
         end
 
         def find
-          return if epic? && group.nil?
+          return if group_relation_without_group?
           return find_diff_commit_user if diff_commit_user?
           return find_diff_commit if diff_commit?
 
@@ -60,7 +60,7 @@ module Gitlab
 
         def prepare_attributes
           attributes.dup.tap do |atts|
-            atts.delete('group') unless epic?
+            atts.delete('group') unless epic? || iteration?
 
             if label?
               atts['type'] = 'ProjectLabel' # Always create project labels
@@ -141,6 +141,10 @@ module Gitlab
           klass == MergeRequestDiffCommit
         end
 
+        def iteration?
+          klass == Iteration
+        end
+
         # If an existing group milestone used the IID
         # claim the IID back and set the group milestone to use one available
         # This is necessary to fix situations like the following:
@@ -157,7 +161,13 @@ module Gitlab
           milestone.ensure_project_iid!
           milestone.save!
         end
+
+        def group_relation_without_group?
+          (epic? || iteration?) && group.nil?
+        end
       end
     end
   end
 end
+
+Gitlab::ImportExport::Project::ObjectBuilder.prepend_mod
diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb
index b40827e36f8..8e91adac196 100644
--- a/lib/gitlab/import_export/project/relation_saver.rb
+++ b/lib/gitlab/import_export/project/relation_saver.rb
@@ -32,7 +32,8 @@ module Gitlab
             project,
             reader.project_tree,
             json_writer,
-            exportable_path: 'project'
+            exportable_path: 'project',
+            current_user: nil
           )
         end
 
diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb
index 6e9548f393a..47196db6f8a 100644
--- a/lib/gitlab/import_export/project/relation_tree_restorer.rb
+++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb
@@ -5,7 +5,7 @@ module Gitlab
     module Project
       class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer
         # Relations which cannot be saved at project level (and have a group assigned)
-        GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze
+        GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze
 
         private
 
diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb
index 1b54e4b975e..bd34cd3ff6e 100644
--- a/lib/gitlab/import_export/project/tree_saver.rb
+++ b/lib/gitlab/import_export/project/tree_saver.rb
@@ -50,7 +50,8 @@ module Gitlab
             reader.project_tree,
             json_writer,
             exportable_path: "project",
-            logger: @logger
+            logger: @logger,
+            current_user: @current_user
           )
 
           Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do