Add latest changes from gitlab-org/gitlab@15-2-stable-eev15.2.0-rc42

16 files changed, 352 insertions, 24 deletions

diff --git a/app/models/concerns/awareness.rb b/app/models/concerns/awareness.rb
new file mode 100644
index 00000000000..da87d87e838
--- /dev/null
+++ b/app/models/concerns/awareness.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Awareness
+  extend ActiveSupport::Concern
+
+  KEY_NAMESPACE = "gitlab:awareness"
+  private_constant :KEY_NAMESPACE
+
+  def join(session)
+    session.join(self)
+
+    nil
+  end
+
+  def leave(session)
+    session.leave(self)
+
+    nil
+  end
+
+  def session_ids
+    with_redis do |redis|
+      redis
+        .smembers(user_sessions_key)
+        # converts session ids from (internal) integer to hex presentation
+        .map { |key| key.to_i.to_s(16) }
+    end
+  end
+
+  private
+
+  def user_sessions_key
+    "#{KEY_NAMESPACE}:user:#{id}:sessions"
+  end
+
+  def with_redis
+    Gitlab::Redis::SharedState.with do |redis|
+      yield redis if block_given?
+    end
+  end
+end
diff --git a/app/models/concerns/cache_markdown_field.rb b/app/models/concerns/cache_markdown_field.rb
index 99dbe464a7c..9ee0fd1db1d 100644
--- a/app/models/concerns/cache_markdown_field.rb
+++ b/app/models/concerns/cache_markdown_field.rb
@@ -172,7 +172,7 @@ module CacheMarkdownField
     refs = all_references(self.author)
 
     references = {}
-    references[:mentioned_users_ids] = refs.mentioned_user_ids.presence
+    references[:mentioned_users_ids] = mentioned_filtered_user_ids_for(refs)
     references[:mentioned_groups_ids] = refs.mentioned_group_ids.presence
     references[:mentioned_projects_ids] = refs.mentioned_project_ids.presence
 
@@ -185,6 +185,13 @@ module CacheMarkdownField
     true
   end
 
+  # Overriden on objects that needs to filter
+  # mentioned users that cannot read them, for example,
+  # guest users that are referenced on a confidential note.
+  def mentioned_filtered_user_ids_for(refs)
+    refs.mentioned_user_ids.presence
+  end
+
   def mentionable_attributes_changed?(changes = saved_changes)
     return false unless is_a?(Mentionable)
 
diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb
index 78340cf967b..fb4ea4206f4 100644
--- a/app/models/concerns/ci/artifactable.rb
+++ b/app/models/concerns/ci/artifactable.rb
@@ -30,6 +30,8 @@ module Ci
         raise NotSupportedAdapterError, 'This file format requires a dedicated adapter'
       end
 
+      ::Gitlab::ApplicationContext.push(artifact: file.model)
+
       file.open do |stream|
         file_format_adapter_class.new(stream).each_blob(&blk)
       end
diff --git a/app/models/concerns/ci/bulk_insertable_tags.rb b/app/models/concerns/ci/bulk_insertable_tags.rb
new file mode 100644
index 00000000000..453b3b3fbc9
--- /dev/null
+++ b/app/models/concerns/ci/bulk_insertable_tags.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Ci
+  module BulkInsertableTags
+    extend ActiveSupport::Concern
+
+    BULK_INSERT_TAG_THREAD_KEY = 'ci_bulk_insert_tags'
+
+    class << self
+      def with_bulk_insert_tags
+        previous = Thread.current[BULK_INSERT_TAG_THREAD_KEY]
+        Thread.current[BULK_INSERT_TAG_THREAD_KEY] = true
+        yield
+      ensure
+        Thread.current[BULK_INSERT_TAG_THREAD_KEY] = previous
+      end
+    end
+
+    # overrides save_tags from acts-as-taggable
+    def save_tags
+      super unless Thread.current[BULK_INSERT_TAG_THREAD_KEY]
+    end
+  end
+end
diff --git a/app/models/concerns/ci/has_status.rb b/app/models/concerns/ci/has_status.rb
index cca66c3ec94..721cb14201f 100644
--- a/app/models/concerns/ci/has_status.rb
+++ b/app/models/concerns/ci/has_status.rb
@@ -23,11 +23,9 @@ module Ci
     UnknownStatusError = Class.new(StandardError)
 
     class_methods do
-      # The parameter `project` is only used for the feature flag check, and will be removed with
-      # https://gitlab.com/gitlab-org/gitlab/-/issues/321972
-      def composite_status(project: nil)
+      def composite_status
         Gitlab::Ci::Status::Composite
-          .new(all, with_allow_failure: columns_hash.key?('allow_failure'), project: project)
+          .new(all, with_allow_failure: columns_hash.key?('allow_failure'))
           .status
       end
 
diff --git a/app/models/concerns/each_batch.rb b/app/models/concerns/each_batch.rb
index 443e1ab53b4..dbc0887dc97 100644
--- a/app/models/concerns/each_batch.rb
+++ b/app/models/concerns/each_batch.rb
@@ -2,6 +2,7 @@
 
 module EachBatch
   extend ActiveSupport::Concern
+  include LooseIndexScan
 
   class_methods do
     # Iterates over the rows in a relation in batches, similar to Rails'
@@ -100,5 +101,65 @@ module EachBatch
         break unless stop
       end
     end
+
+    # Iterates over the rows in a relation in batches by skipping duplicated values in the column.
+    # Example: counting the number of distinct authors in `issues`
+    #
+    #  - Table size: 100_000
+    #  - Column: author_id
+    #  - Distinct author_ids in the table: 1000
+    #
+    #  The query will read maximum 1000 rows if we have index coverage on user_id.
+    #
+    #  > count = 0
+    #  > Issue.distinct_each_batch(column: 'author_id', of: 1000) { |r| count += r.count(:author_id) }
+    def distinct_each_batch(column:, order: :asc, of: 1000)
+      start = except(:select)
+        .select(column)
+        .reorder(column => order)
+
+      start = start.take
+
+      return unless start
+
+      start_id = start[column]
+      arel_table = self.arel_table
+      arel_column = arel_table[column.to_s]
+
+      1.step do |index|
+        stop = loose_index_scan(column: column, order: order) do |cte_query, inner_query|
+          if order == :asc
+            [cte_query.where(arel_column.gteq(start_id)), inner_query]
+          else
+            [cte_query.where(arel_column.lteq(start_id)), inner_query]
+          end
+        end.offset(of).take
+
+        if stop
+          stop_id = stop[column]
+
+          relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query|
+            if order == :asc
+              [cte_query.where(arel_column.gteq(start_id)), inner_query.where(arel_column.lt(stop_id))]
+            else
+              [cte_query.where(arel_column.lteq(start_id)), inner_query.where(arel_column.gt(stop_id))]
+            end
+          end
+          start_id = stop_id
+        else
+          relation = loose_index_scan(column: column, order: order) do |cte_query, inner_query|
+            if order == :asc
+              [cte_query.where(arel_column.gteq(start_id)), inner_query]
+            else
+              [cte_query.where(arel_column.lteq(start_id)), inner_query]
+            end
+          end
+        end
+
+        unscoped { yield relation, index }
+
+        break unless stop
+      end
+    end
   end
 end
diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb
index 445277a7a7c..ecb120d8013 100644
--- a/app/models/concerns/enums/ci/commit_status.rb
+++ b/app/models/concerns/enums/ci/commit_status.rb
@@ -29,9 +29,12 @@ module Enums
           builds_disabled: 20,
           environment_creation_failure: 21,
           deployment_rejected: 22,
+          protected_environment_failure: 1_000,
           insufficient_bridge_permissions: 1_001,
           downstream_bridge_project_not_found: 1_002,
           invalid_bridge_trigger: 1_003,
+          upstream_bridge_project_not_found: 1_004,
+          insufficient_upstream_permissions: 1_005,
           bridge_pipeline_is_child_pipeline: 1_006, # not used anymore, but cannot be deleted because of old data
           downstream_pipeline_creation_failed: 1_007,
           secrets_provider_not_found: 1_008,
@@ -42,5 +45,3 @@ module Enums
     end
   end
 end
-
-Enums::Ci::CommitStatus.prepend_mod_with('Enums::Ci::CommitStatus')
diff --git a/app/models/concerns/integrations/has_issue_tracker_fields.rb b/app/models/concerns/integrations/has_issue_tracker_fields.rb
index b1def38d019..57f8e21c5a6 100644
--- a/app/models/concerns/integrations/has_issue_tracker_fields.rb
+++ b/app/models/concerns/integrations/has_issue_tracker_fields.rb
@@ -5,26 +5,32 @@ module Integrations
     extend ActiveSupport::Concern
 
     included do
+      self.field_storage = :data_fields
+
       field :project_url,
             required: true,
-            storage: :data_fields,
             title: -> { _('Project URL') },
-            help: -> { s_('IssueTracker|The URL to the project in the external issue tracker.') }
+            help: -> do
+              s_('IssueTracker|The URL to the project in the external issue tracker.')
+            end
 
       field :issues_url,
             required: true,
-            storage: :data_fields,
             title: -> { s_('IssueTracker|Issue URL') },
             help: -> do
-              format s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.'),
+              ERB::Util.html_escape(
+                s_('IssueTracker|The URL to view an issue in the external issue tracker. Must contain %{colon_id}.')
+              ) % {
                 colon_id: '<code>:id</code>'.html_safe
+              }
             end
 
       field :new_issue_url,
             required: true,
-            storage: :data_fields,
             title: -> { s_('IssueTracker|New issue URL') },
-            help: -> { s_('IssueTracker|The URL to create an issue in the external issue tracker.') }
+            help: -> do
+              s_('IssueTracker|The URL to create an issue in the external issue tracker.')
+            end
     end
   end
 end
diff --git a/app/models/concerns/integrations/slack_mattermost_notifier.rb b/app/models/concerns/integrations/slack_mattermost_notifier.rb
index 3bdaa852ddf..142e62bb501 100644
--- a/app/models/concerns/integrations/slack_mattermost_notifier.rb
+++ b/app/models/concerns/integrations/slack_mattermost_notifier.rb
@@ -34,7 +34,7 @@ module Integrations
     class HTTPClient
       def self.post(uri, params = {})
         params.delete(:http_options) # these are internal to the client and we do not want them
-        Gitlab::HTTP.post(uri, body: params, use_read_total_timeout: true)
+        Gitlab::HTTP.post(uri, body: params)
       end
     end
   end
diff --git a/app/models/concerns/loose_index_scan.rb b/app/models/concerns/loose_index_scan.rb
new file mode 100644
index 00000000000..5d37a30171a
--- /dev/null
+++ b/app/models/concerns/loose_index_scan.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module LooseIndexScan
+  extend ActiveSupport::Concern
+
+  class_methods do
+    # Builds a recursive query to read distinct values from a column.
+    #
+    # Example 1: collect all distinct author ids for the `issues` table
+    #
+    # Bad: The DB reads all issues, sorts and dedups them in memory
+    #
+    # > Issue.select(:author_id).distinct.map(&:author_id)
+    #
+    # Good: Use loose index scan (skip index scan)
+    #
+    # > Issue.loose_index_scan(column: :author_id).map(&:author_id)
+    #
+    # Example 2: List of users for the DONE todos selector. Select all users who created a todo.
+    #
+    # Bad: Loads all DONE todos for the given user and extracts the author_ids
+    #
+    # > User.where(id: Todo.where(user_id: 4156052).done.select(:author_id))
+    #
+    # Good: Loads distinct author_ids from todos and then loads users
+    #
+    # > distinct_authors = Todo.where(user_id: 4156052).done.loose_index_scan(column: :author_id).select(:author_id)
+    # > User.where(id: distinct_authors)
+    def loose_index_scan(column:, order: :asc)
+      arel_table = self.arel_table
+      arel_column = arel_table[column.to_s]
+
+      cte = Gitlab::SQL::RecursiveCTE.new(:loose_index_scan_cte, union_args: { remove_order: false })
+
+      cte_query = except(:select)
+        .select(column)
+        .order(column => order)
+        .limit(1)
+
+      inner_query = except(:select)
+
+      cte_query, inner_query = yield([cte_query, inner_query]) if block_given?
+      cte << cte_query
+
+      inner_query = if order == :asc
+                      inner_query.where(arel_column.gt(cte.table[column.to_s]))
+                    else
+                      inner_query.where(arel_column.lt(cte.table[column.to_s]))
+                    end
+
+      inner_query = inner_query.order(column => order)
+        .select(column)
+        .limit(1)
+
+      cte << cte.table
+        .project(Arel::Nodes::Grouping.new(Arel.sql(inner_query.to_sql)).as(column.to_s))
+
+      unscoped do
+        select(column)
+          .with
+          .recursive(cte.to_arel)
+          .from(cte.alias_to(arel_table))
+          .where(arel_column.not_eq(nil)) # filtering out the last NULL value
+      end
+    end
+  end
+end
diff --git a/app/models/concerns/milestoneable.rb b/app/models/concerns/milestoneable.rb
index 12041b103f6..14c54d99ef3 100644
--- a/app/models/concerns/milestoneable.rb
+++ b/app/models/concerns/milestoneable.rb
@@ -16,7 +16,7 @@ module Milestoneable
 
     scope :any_milestone, -> { where.not(milestone_id: nil) }
     scope :with_milestone, ->(title) { left_joins_milestones.where(milestones: { title: title }) }
-    scope :without_particular_milestone, ->(title) { left_outer_joins(:milestone).where("milestones.title != ? OR milestone_id IS NULL", title) }
+    scope :without_particular_milestones, ->(titles) { left_outer_joins(:milestone).where("milestones.title NOT IN (?) OR milestone_id IS NULL", titles) }
     scope :any_release, -> { joins_milestone_releases }
     scope :with_release, -> (tag, project_id) { joins_milestone_releases.where( milestones: { releases: { tag: tag, project_id: project_id } } ) }
     scope :without_particular_release, -> (tag, project_id) { joins_milestone_releases.where.not(milestones: { releases: { tag: tag, project_id: project_id } }) }
diff --git a/app/models/concerns/notification_branch_selection.rb b/app/models/concerns/notification_branch_selection.rb
index 18ec996c3df..f2df7579a65 100644
--- a/app/models/concerns/notification_branch_selection.rb
+++ b/app/models/concerns/notification_branch_selection.rb
@@ -6,13 +6,15 @@
 module NotificationBranchSelection
   extend ActiveSupport::Concern
 
-  def branch_choices
-    [
-      [_('All branches'), 'all'].freeze,
-      [_('Default branch'), 'default'].freeze,
-      [_('Protected branches'), 'protected'].freeze,
-      [_('Default branch and protected branches'), 'default_and_protected'].freeze
-    ].freeze
+  class_methods do
+    def branch_choices
+      [
+        [_('All branches'), 'all'].freeze,
+        [_('Default branch'), 'default'].freeze,
+        [_('Protected branches'), 'protected'].freeze,
+        [_('Default branch and protected branches'), 'default_and_protected'].freeze
+      ].freeze
+    end
   end
 
   def notify_for_branch?(data)
diff --git a/app/models/concerns/packages/fips.rb b/app/models/concerns/packages/fips.rb
new file mode 100644
index 00000000000..b8589cdc991
--- /dev/null
+++ b/app/models/concerns/packages/fips.rb
@@ -0,0 +1,11 @@
+# rubocop:disable Naming/FileName
+# frozen_string_literal: true
+
+module Packages
+  module FIPS
+    extend ActiveSupport::Concern
+
+    DisabledError = Class.new(StandardError)
+  end
+end
+# rubocop:enable Naming/FileName
diff --git a/app/models/concerns/participable.rb b/app/models/concerns/participable.rb
index 20743ebcb52..f59b5d1ecc8 100644
--- a/app/models/concerns/participable.rb
+++ b/app/models/concerns/participable.rb
@@ -92,7 +92,13 @@ module Participable
   end
 
   def raw_participants(current_user = nil, verify_access: false)
-    ext = Gitlab::ReferenceExtractor.new(project, current_user)
+    extractor = Gitlab::ReferenceExtractor.new(project, current_user)
+
+    # Used to extract references from confidential notes.
+    # Referenced users that cannot read confidential notes are
+    # later removed from participants array.
+    internal_notes_extractor = Gitlab::ReferenceExtractor.new(project, current_user)
+
     participants = Set.new
     process = [self]
 
@@ -107,6 +113,8 @@ module Participable
 
         source.class.participant_attrs.each do |attr|
           if attr.respond_to?(:call)
+            ext = use_internal_notes_extractor_for?(source) ? internal_notes_extractor : extractor
+
             source.instance_exec(current_user, ext, &attr)
           else
             process << source.__send__(attr) # rubocop:disable GitlabSecurity/PublicSend
@@ -121,7 +129,18 @@ module Participable
       end
     end
 
-    participants.merge(ext.users)
+    participants.merge(users_that_can_read_internal_notes(internal_notes_extractor))
+    participants.merge(extractor.users)
+  end
+
+  def use_internal_notes_extractor_for?(source)
+    source.is_a?(Note) && source.confidential?
+  end
+
+  def users_that_can_read_internal_notes(extractor)
+    return [] unless self.is_a?(Noteable) && self.try(:resource_parent)
+
+    Ability.users_that_can_read_internal_notes(extractor.users, self.resource_parent)
   end
 
   def source_visible_to_user?(source, user)
diff --git a/app/models/concerns/require_email_verification.rb b/app/models/concerns/require_email_verification.rb
new file mode 100644
index 00000000000..cf6a31e6ebd
--- /dev/null
+++ b/app/models/concerns/require_email_verification.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+# == Require Email Verification module
+#
+# Contains functionality to handle email verification
+module RequireEmailVerification
+  extend ActiveSupport::Concern
+  include Gitlab::Utils::StrongMemoize
+
+  # This value is twice the amount we want it to be, because due to a bug in the devise-two-factor
+  # gem every failed login attempt increments the value of failed_attempts by 2 instead of 1.
+  # See: https://github.com/tinfoil/devise-two-factor/issues/127
+  MAXIMUM_ATTEMPTS = 3 * 2
+  UNLOCK_IN = 24.hours
+
+  included do
+    # Virtual attribute for the email verification token form
+    attr_accessor :verification_token
+  end
+
+  # When overridden, do not send Devise unlock instructions when locking access.
+  def lock_access!(opts = {})
+    return super unless override_devise_lockable?
+
+    super({ send_instructions: false })
+  end
+
+  protected
+
+  # We cannot override the class methods `maximum_attempts` and `unlock_in`, because we want to
+  # check for 2FA being enabled on the instance. So instead override the Devise Lockable methods
+  # where those values are used.
+  def attempts_exceeded?
+    return super unless override_devise_lockable?
+
+    failed_attempts >= MAXIMUM_ATTEMPTS
+  end
+
+  def lock_expired?
+    return super unless override_devise_lockable?
+
+    locked_at && locked_at < UNLOCK_IN.ago
+  end
+
+  private
+
+  def override_devise_lockable?
+    strong_memoize(:override_devise_lockable) do
+      Feature.enabled?(:require_email_verification, self) && !two_factor_enabled?
+    end
+  end
+end
diff --git a/app/models/concerns/vulnerability_finding_helpers.rb b/app/models/concerns/vulnerability_finding_helpers.rb
index 7f96b3901f1..4cf36f83857 100644
--- a/app/models/concerns/vulnerability_finding_helpers.rb
+++ b/app/models/concerns/vulnerability_finding_helpers.rb
@@ -42,4 +42,41 @@ module VulnerabilityFindingHelpers
       )
     end
   end
+
+  def build_vulnerability_finding(security_finding)
+    report_finding = report_finding_for(security_finding)
+    return Vulnerabilities::Finding.new unless report_finding
+
+    finding_data = report_finding.to_hash.except(:compare_key, :identifiers, :location, :scanner, :links, :signatures,
+                                                 :flags, :evidence)
+    identifiers = report_finding.identifiers.map do |identifier|
+      Vulnerabilities::Identifier.new(identifier.to_hash)
+    end
+    signatures = report_finding.signatures.map do |signature|
+      Vulnerabilities::FindingSignature.new(signature.to_hash)
+    end
+    evidence = Vulnerabilities::Finding::Evidence.new(data: report_finding.evidence.data) if report_finding.evidence
+
+    Vulnerabilities::Finding.new(finding_data).tap do |finding|
+      finding.location_fingerprint = report_finding.location.fingerprint
+      finding.vulnerability = vulnerability_for(security_finding.uuid)
+      finding.project = project
+      finding.sha = pipeline.sha
+      finding.scanner = security_finding.scanner
+      finding.finding_evidence = evidence
+
+      if calculate_false_positive?
+        finding.vulnerability_flags = report_finding.flags.map do |flag|
+          Vulnerabilities::Flag.new(flag)
+        end
+      end
+
+      finding.identifiers = identifiers
+      finding.signatures = signatures
+    end
+  end
+
+  def calculate_false_positive?
+    project.licensed_feature_available?(:sast_fp_reduction)
+  end
 end