Add latest changes from gitlab-org/gitlab@15-9-stable-eev15.9.0-rc42

7 files changed, 177 insertions, 95 deletions

diff --git a/tooling/danger/config_files.rb b/tooling/danger/config_files.rb
index 914605a3783..e165792471f 100644
--- a/tooling/danger/config_files.rb
+++ b/tooling/danger/config_files.rb
@@ -1,15 +1,14 @@
 # frozen_string_literal: true
 
 require 'yaml'
+require_relative 'suggestor'
 
 module Tooling
   module Danger
     module ConfigFiles
-      SUGGEST_INTRODUCED_BY_COMMENT = <<~SUGGEST_COMMENT
-        ```suggestion
-        introduced_by_url: %<url>
-        ```
-      SUGGEST_COMMENT
+      include ::Tooling::Danger::Suggestor
+
+      MISSING_INTRODUCED_BY_REGEX = /^\+?(?<attr_name>\s*introduced_by_url):\s*$/.freeze
 
       CONFIG_DIRS = %w[
         config/feature_flags
@@ -18,14 +17,12 @@ module Tooling
       ].freeze
 
       def add_suggestion_for_missing_introduced_by_url
-        new_config_files.each do |file_name|
-          config_file_lines = project_helper.file_lines(file_name)
-
-          config_file_lines.each_with_index do |added_line, i|
-            next unless added_line =~ /^introduced_by_url:\s?$/
-
-            markdown(format(SUGGEST_INTRODUCED_BY_COMMENT, url: helper.mr_web_url), file: file_name, line: i + 1)
-          end
+        new_config_files.each do |filename|
+          add_suggestion(
+            filename: filename,
+            regex: MISSING_INTRODUCED_BY_REGEX,
+            replacement: "\\k<attr_name>: #{helper.mr_web_url}"
+          )
         end
       end
 
diff --git a/tooling/danger/feature_flag.rb b/tooling/danger/feature_flag.rb
index cef64e52af3..da0b7053af1 100644
--- a/tooling/danger/feature_flag.rb
+++ b/tooling/danger/feature_flag.rb
@@ -16,26 +16,22 @@ module Tooling
       end
 
       class Found
+        ATTRIBUTES = %w[name introduced_by_url rollout_issue_url milestone type group default_enabled].freeze
+
         attr_reader :path
 
         def initialize(path)
           @path = path
         end
 
-        def raw
-          @raw ||= File.read(path)
-        end
-
-        def group
-          @group ||= yaml['group']
+        ATTRIBUTES.each do |attribute|
+          define_method(attribute) do
+            yaml[attribute]
+          end
         end
 
-        def default_enabled
-          @default_enabled ||= yaml['default_enabled']
-        end
-
-        def rollout_issue_url
-          @rollout_issue_url ||= yaml['rollout_issue_url']
+        def raw
+          @raw ||= File.read(path)
         end
 
         def group_match_mr_label?(mr_group_label)
diff --git a/tooling/danger/product_intelligence.rb b/tooling/danger/product_intelligence.rb
index 58e327408a1..d25f966504f 100644
--- a/tooling/danger/product_intelligence.rb
+++ b/tooling/danger/product_intelligence.rb
@@ -22,6 +22,11 @@ module Tooling
 
       MSG
 
+      CHANGED_USAGE_DATA_MESSAGE = <<~MSG
+        Notice that implementing metrics directly in usage_data.rb has been deprecated. ([Deprecated Usage Metrics](https://docs.gitlab.com/ee/development/service_ping/usage_data.html#usage-data-metrics-guide))
+        Please use [Instrumentation Classes](https://docs.gitlab.com/ee/development/service_ping/metrics_instrumentation.html) instead.
+      MSG
+
       WORKFLOW_LABELS = [
         APPROVED_LABEL,
         REVIEW_LABEL
@@ -47,6 +52,13 @@ module Tooling
         helper.labels_to_add.concat(missing_labels) unless missing_labels.empty?
       end
 
+      def check_usage_data_insertions!
+        usage_data_changes = helper.changed_lines("lib/gitlab/usage_data.rb")
+        return if usage_data_changes.none? { |change| change.start_with?("+") }
+
+        warn format(CHANGED_USAGE_DATA_MESSAGE)
+      end
+
       private
 
       def convert_to_table(items)
diff --git a/tooling/danger/project_helper.rb b/tooling/danger/project_helper.rb
index fbf102422aa..2a77ac337a2 100644
--- a/tooling/danger/project_helper.rb
+++ b/tooling/danger/project_helper.rb
@@ -12,6 +12,7 @@ module Tooling
         sidekiq_queues
         specialization_labels
         specs
+        stable_branch_patch
         z_metadata
       ].freeze
 
diff --git a/tooling/danger/specs.rb b/tooling/danger/specs.rb
index 6c0459a4344..f95a798d53e 100644
--- a/tooling/danger/specs.rb
+++ b/tooling/danger/specs.rb
@@ -1,12 +1,14 @@
 # frozen_string_literal: true
 
+require_relative 'suggestor'
+
 module Tooling
   module Danger
     module Specs
+      include ::Tooling::Danger::Suggestor
+
       SPEC_FILES_REGEX = 'spec/'
       EE_PREFIX = 'ee/'
-      MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[[^\]]+)/.freeze
-      MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
 
       PROJECT_FACTORIES = %w[
         :project
@@ -29,22 +31,18 @@ module Tooling
       /x.freeze
 
       PROJECT_FACTORY_REPLACEMENT = '\k<head>let_it_be\k<tail>'
-      SUGGESTION_MARKDOWN = <<~SUGGESTION_MARKDOWN
-      ```suggestion
-      %<suggested_line>s
-      ```
-      SUGGESTION_MARKDOWN
-
-      MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
-      If order of the result is not important, please consider using `match_array` to avoid flakiness.
-      SUGGEST_COMMENT
-
       PROJECT_FACTORY_SUGGESTION = <<~SUGGEST_COMMENT
       Project creations are very slow. Use `let_it_be`, `build` or `build_stubbed` if possible.
       See [testing best practices](https://docs.gitlab.com/ee/development/testing_guide/best_practices.html#optimize-factory-usage)
       for background information and alternative options.
       SUGGEST_COMMENT
 
+      MATCH_WITH_ARRAY_REGEX = /(?<to>to\(?\s*)(?<matcher>match|eq)(?<expectation>[( ]?\[(?=.*,)[^\]]+)/.freeze
+      MATCH_WITH_ARRAY_REPLACEMENT = '\k<to>match_array\k<expectation>'
+      MATCH_WITH_ARRAY_SUGGESTION = <<~SUGGEST_COMMENT
+      If order of the result is not important, please consider using `match_array` to avoid flakiness.
+      SUGGEST_COMMENT
+
       RSPEC_TOP_LEVEL_DESCRIBE_REGEX = /^\+.?RSpec\.describe(.+)/.freeze
       FEATURE_CATEGORY_SUGGESTION = <<~SUGGESTION_MARKDOWN
       Consider adding `feature_category: <feature_category_name>` for this example if it is not set already.
@@ -69,19 +67,19 @@ module Tooling
 
       def add_suggestions_for_match_with_array(filename)
         add_suggestion(
-          filename,
-          MATCH_WITH_ARRAY_REGEX,
-          MATCH_WITH_ARRAY_SUGGESTION,
-          MATCH_WITH_ARRAY_REPLACEMENT
+          filename: filename,
+          regex: MATCH_WITH_ARRAY_REGEX,
+          replacement: MATCH_WITH_ARRAY_REPLACEMENT,
+          comment_text: MATCH_WITH_ARRAY_SUGGESTION
         )
       end
 
       def add_suggestions_for_project_factory_usage(filename)
         add_suggestion(
-          filename,
-          PROJECT_FACTORY_REGEX,
-          PROJECT_FACTORY_SUGGESTION,
-          PROJECT_FACTORY_REPLACEMENT
+          filename: filename,
+          regex: PROJECT_FACTORY_REGEX,
+          replacement: PROJECT_FACTORY_REPLACEMENT,
+          comment_text: PROJECT_FACTORY_SUGGESTION
         )
       end
 
@@ -103,53 +101,9 @@ module Tooling
 
           suggested_line = file_lines[line_number]
 
-          text = format(comment(FEATURE_CATEGORY_SUGGESTION), suggested_line: suggested_line)
-          markdown(text, file: filename, line: line_number + 1)
+          markdown(comment(FEATURE_CATEGORY_SUGGESTION, suggested_line), file: filename, line: line_number.succ)
         end
       end
-
-      private
-
-      def added_lines_matching(filename, regex)
-        helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex)
-      end
-
-      def add_suggestion(filename, regex, comment_text, replacement = nil, exclude = nil)
-        added_lines = added_lines_matching(filename, regex)
-
-        return if added_lines.empty?
-
-        spec_file_lines = project_helper.file_lines(filename)
-
-        added_lines.each_with_object([]) do |added_line, processed_line_numbers|
-          line_number = find_line_number(spec_file_lines, added_line.delete_prefix('+'), exclude_indexes: processed_line_numbers)
-          next unless line_number
-          next if !exclude.nil? && added_line.include?(exclude)
-
-          processed_line_numbers << line_number
-
-          suggested_line = spec_file_lines[line_number]
-          suggested_line = suggested_line.gsub(regex, replacement) unless replacement.nil?
-
-          text = format(comment(comment_text), suggested_line: suggested_line)
-          markdown(text, file: filename, line: line_number.succ)
-        end
-      end
-
-      def comment(comment_text)
-        <<~COMMENT_BODY.chomp
-        #{SUGGESTION_MARKDOWN}
-        #{comment_text}
-        COMMENT_BODY
-      end
-
-      def find_line_number(file_lines, searched_line, exclude_indexes: [])
-        _, index = file_lines.each_with_index.find do |file_line, index|
-          file_line == searched_line && !exclude_indexes.include?(index)
-        end
-
-        index
-      end
     end
   end
 end
diff --git a/tooling/danger/stable_branch.rb b/tooling/danger/stable_branch.rb
index 6c0b94b4f06..8fac1cc5fbf 100644
--- a/tooling/danger/stable_branch.rb
+++ b/tooling/danger/stable_branch.rb
@@ -6,6 +6,7 @@ module Tooling
       VersionApiError = Class.new(StandardError)
 
       STABLE_BRANCH_REGEX = %r{\A(?<version>\d+-\d+)-stable-ee\z}.freeze
+      FAILING_PACKAGE_AND_TEST_STATUSES = %w[manual canceled].freeze
 
       # rubocop:disable Lint/MixedRegexpCaptureTypes
       VERSION_REGEX = %r{
@@ -32,26 +33,69 @@ module Tooling
       This branch is meant for backporting bug fixes. If this MR qualifies please add the `type::bug` label. #{MAINTENANCE_POLICY_MESSAGE}
       MSG
 
-      VERSION_ERROR_MESSAGE = <<~MSG
-      Patches are only being accepted on the most recent 3 minor versions of GitLab. #{MAINTENANCE_POLICY_MESSAGE}
+      VERSION_WARNING_MESSAGE = <<~MSG
+      Backporting to older releases requires an [exception request process](https://docs.gitlab.com/ee/policy/maintenance.html#backporting-to-older-releases)
       MSG
 
       FAILED_VERSION_REQUEST_MESSAGE = <<~MSG
       There was a problem checking if this is a qualified version for backporting. Re-running this job may fix the problem.
       MSG
 
+      PIPELINE_EXPEDITE_ERROR_MESSAGE = <<~MSG
+      ~"pipeline:expedite" is not allowed on stable branches because it causes the `e2e:package-and-test` job to be skipped.
+      MSG
+
+      NEEDS_PACKAGE_AND_TEST_MESSAGE = <<~MSG
+      The `e2e:package-and-test` job is not present or needs to be triggered manually. Please start the `e2e:package-and-test`
+      job and re-run `danger-review`.
+      MSG
+
+      WARN_PACKAGE_AND_TEST_MESSAGE = <<~MSG
+      The `e2e:package-and-test` job needs to succeed or have approval from a Software Engineer in Test. See the section below
+      for more details.
+      MSG
+
       # rubocop:disable Style/SignalException
       def check!
-        return unless stable_target_branch && !helper.security_mr?
+        return unless non_security_stable_branch?
 
         fail FEATURE_ERROR_MESSAGE if has_feature_label?
         fail BUG_ERROR_MESSAGE unless has_bug_label?
-        fail VERSION_ERROR_MESSAGE unless targeting_patchable_version?
+
+        warn VERSION_WARNING_MESSAGE unless targeting_patchable_version?
+
+        return if has_flaky_failure_label? || has_only_documentation_changes?
+
+        fail PIPELINE_EXPEDITE_ERROR_MESSAGE if has_pipeline_expedite_label?
+
+        status = package_and_test_status
+
+        if status.nil? || FAILING_PACKAGE_AND_TEST_STATUSES.include?(status) # rubocop:disable Style/GuardClause
+          fail NEEDS_PACKAGE_AND_TEST_MESSAGE
+        else
+          warn WARN_PACKAGE_AND_TEST_MESSAGE unless status == 'success'
+        end
       end
       # rubocop:enable Style/SignalException
 
+      def non_security_stable_branch?
+        !!stable_target_branch && !helper.security_mr?
+      end
+
       private
 
+      def package_and_test_status
+        mr_head_pipeline_id = gitlab.mr_json.dig('head_pipeline', 'id')
+        return unless mr_head_pipeline_id
+
+        pipeline_bridges = gitlab.api.pipeline_bridges(helper.mr_target_project_id, mr_head_pipeline_id)
+        package_and_test_pipeline = pipeline_bridges&.find { |j| j['name'] == 'e2e:package-and-test' }
+
+        return unless package_and_test_pipeline
+
+        package_and_test_pipeline['status']
+      end
+
       def stable_target_branch
         helper.mr_target_branch.match(STABLE_BRANCH_REGEX)
       end
@@ -64,12 +108,27 @@ module Tooling
         helper.mr_has_labels?('type::bug')
       end
 
+      def has_pipeline_expedite_label?
+        helper.mr_has_labels?('pipeline:expedite')
+      end
+
+      def has_flaky_failure_label?
+        helper.mr_has_labels?('failure::flaky-test')
+      end
+
+      def has_only_documentation_changes?
+        categories_changed = helper.changes_by_category.keys
+        return false unless categories_changed.size == 1
+        return true if categories_changed.first == :docs
+
+        false
+      end
+
       def targeting_patchable_version?
         raise VersionApiError if last_three_minor_versions.empty?
 
         last_three_minor_versions.include?(targeted_version)
       rescue VersionApiError
-        # don't fail the job since we do not know the recent versions
         warn FAILED_VERSION_REQUEST_MESSAGE
         true
       end
diff --git a/tooling/danger/suggestor.rb b/tooling/danger/suggestor.rb
new file mode 100644
index 00000000000..ffda98e67d0
--- /dev/null
+++ b/tooling/danger/suggestor.rb
@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+module Tooling
+  module Danger
+    module Suggestor
+      # For file lines matching `regex` adds suggestion `replacement` with `comment_text` added.
+      def add_suggestion(filename:, regex:, replacement: nil, comment_text: nil, exclude: nil)
+        added_lines = added_lines_matching(filename, regex)
+
+        return if added_lines.empty?
+
+        file_lines = project_helper.file_lines(filename)
+
+        added_lines.each_with_object([]) do |added_line, processed_line_numbers|
+          line_number = find_line_number(file_lines, added_line.delete_prefix('+'),
+exclude_indexes: processed_line_numbers)
+
+          next unless line_number
+          next if !exclude.nil? && added_line.include?(exclude)
+
+          processed_line_numbers << line_number
+
+          if replacement
+            suggestion_text = file_lines[line_number]
+            suggestion_text = suggestion_text.gsub(regex, replacement)
+          end
+
+          markdown(comment(comment_text, suggestion_text), file: filename, line: line_number.succ)
+        end
+      end
+
+      private
+
+      def added_lines_matching(filename, regex)
+        helper.changed_lines(filename).grep(/\A\+( )?/).grep(regex)
+      end
+
+      def find_line_number(file_lines, searched_line, exclude_indexes: [])
+        _, index = file_lines.each_with_index.find do |file_line, index|
+          file_line == searched_line && !exclude_indexes.include?(index) # rubocop:disable Rails/NegateInclude
+        end
+
+        index
+      end
+
+      def comment(comment_text = nil, suggested_line = nil)
+        if suggested_line
+          suggestion_text = <<~SUGGESTION
+            ```suggestion
+            %<suggested_line>s
+            ```
+          SUGGESTION
+        end
+
+        comment_body = <<~COMMENT_BODY.chomp
+        #{suggestion_text}
+        #{comment_text}
+        COMMENT_BODY
+
+        format(comment_body.chomp, suggested_line: suggested_line)
+      end
+    end
+  end
+end