Add latest changes from gitlab-org/gitlab@14-3-stable-eev14.3.0-rc42

39 files changed, 527 insertions, 95 deletions

diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb
index d576953c1a0..3cfed8e5e2c 100644
--- a/lib/gitlab/ci/artifact_file_reader.rb
+++ b/lib/gitlab/ci/artifact_file_reader.rb
@@ -45,7 +45,7 @@ module Gitlab
       end
 
       def read_zip_file!(file_path)
-        if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: false)
+        if ::Feature.enabled?(:ci_new_artifact_file_reader, job.project, default_enabled: :yaml)
           read_with_new_artifact_file_reader(file_path)
         else
           read_with_legacy_artifact_file_reader(file_path)
diff --git a/lib/gitlab/ci/config/entry/default.rb b/lib/gitlab/ci/config/entry/default.rb
index eaaf9f69102..145772c7a92 100644
--- a/lib/gitlab/ci/config/entry/default.rb
+++ b/lib/gitlab/ci/config/entry/default.rb
@@ -53,7 +53,7 @@ module Gitlab
             description: 'Set retry default value.',
             inherit: false
 
-          entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings,
+          entry :tags, Entry::Tags,
             description: 'Set the default tags.',
             inherit: false
 
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index bd4d5f33689..f867189d521 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -85,7 +85,7 @@ module Gitlab
             description: 'Retry configuration for this job.',
             inherit: true
 
-          entry :tags, ::Gitlab::Config::Entry::ArrayOfStrings,
+          entry :tags, Entry::Tags,
             description: 'Set the tags.',
             inherit: true
 
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 3543b5493bd..2549c35ebd6 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -16,6 +16,7 @@ module Gitlab
 
           PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables
                                         inherit allow_failure when needs resource_group].freeze
+          MAX_NESTING_LEVEL = 10
 
           included do
             validations do
@@ -31,7 +32,7 @@ module Gitlab
 
               with_options allow_nil: true do
                 validates :extends, array_of_strings_or_string: true
-                validates :rules, nested_array_of_hashes: true
+                validates :rules, nested_array_of_hashes_or_arrays: { max_level: MAX_NESTING_LEVEL }
                 validates :resource_group, type: String
               end
             end
diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb
new file mode 100644
index 00000000000..ca3b48372e2
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/tags.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Config
+      module Entry
+        ##
+        # Entry that represents an array of tags.
+        #
+        class Tags < ::Gitlab::Config::Entry::Node
+          include ::Gitlab::Config::Entry::Validatable
+
+          TAGS_LIMIT = 50
+
+          validations do
+            validates :config, array_of_strings: true
+
+            validate do
+              next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml)
+
+              if config.is_a?(Array) && config.size >= TAGS_LIMIT
+                errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT })
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/cron_parser.rb b/lib/gitlab/ci/cron_parser.rb
index bc03658aab8..7334a112ccf 100644
--- a/lib/gitlab/ci/cron_parser.rb
+++ b/lib/gitlab/ci/cron_parser.rb
@@ -6,8 +6,40 @@ module Gitlab
       VALID_SYNTAX_SAMPLE_TIME_ZONE = 'UTC'
       VALID_SYNTAX_SAMPLE_CRON = '* * * * *'
 
-      def self.parse_natural(expression, cron_timezone = 'UTC')
-        new(Fugit::Nat.parse(expression)&.original, cron_timezone)
+      class << self
+        def parse_natural(expression, cron_timezone = 'UTC')
+          new(Fugit::Nat.parse(expression)&.original, cron_timezone)
+        end
+
+        # This method generates compatible expressions that can be
+        # parsed by Fugit::Nat.parse to generate a cron line.
+        # It takes start date of the cron and cadence in the following format:
+        # cadence = {
+        #   unit: 'day/week/month/year'
+        #   duration: 1
+        # }
+        def parse_natural_with_timestamp(starts_at, cadence)
+          case cadence[:unit]
+          when 'day' # Currently supports only 'every 1 day'.
+            "#{starts_at.min} #{starts_at.hour} * * *"
+          when 'week' # Currently supports only 'every 1 week'.
+            "#{starts_at.min} #{starts_at.hour} * * #{starts_at.wday}"
+          when 'month'
+            unless [1, 3, 6, 12].include?(cadence[:duration])
+              raise NotImplementedError, "The cadence #{cadence} is not supported"
+            end
+
+            "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{fall_in_months(cadence[:duration], starts_at)} *"
+          when 'year' # Currently supports only 'every 1 year'.
+            "#{starts_at.min} #{starts_at.hour} #{starts_at.mday} #{starts_at.month} *"
+          else
+            raise NotImplementedError, "The cadence unit #{cadence[:unit]} is not implemented"
+          end
+        end
+
+        def fall_in_months(offset, start_date)
+          (1..(12 / offset)).map { |i| start_date.next_month(offset * i).month }.join(',')
+        end
       end
 
       def initialize(cron, cron_timezone = 'UTC')
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index cd2c135dd7e..8c1067b9bc6 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -21,7 +21,7 @@ module Gitlab
       def initialize(project:, current_user:, sha: nil)
         @project = project
         @current_user = current_user
-        @sha = sha || project.repository.commit&.sha
+        @sha = sha || project&.repository&.commit&.sha
       end
 
       def validate(content, dry_run: false)
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index 41acb4d5040..1cf4f252ab9 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -86,6 +86,7 @@ module Gitlab
 
           def create_finding(data, remediations = [])
             identifiers = create_identifiers(data['identifiers'])
+            flags = create_flags(data['flags'])
             links = create_links(data['links'])
             location = create_location(data['location'] || {})
             signatures = create_signatures(tracking_data(data))
@@ -111,6 +112,7 @@ module Gitlab
                 scanner: create_scanner(data['scanner']),
                 scan: report&.scan,
                 identifiers: identifiers,
+                flags: flags,
                 links: links,
                 remediations: remediations,
                 raw_metadata: data.to_json,
@@ -205,6 +207,18 @@ module Gitlab
                 url: identifier['url']))
           end
 
+          def create_flags(flags)
+            return [] unless flags.is_a?(Array)
+
+            flags.map { |flag| create_flag(flag) }.compact
+          end
+
+          def create_flag(flag)
+            return unless flag.is_a?(Hash)
+
+            ::Gitlab::Ci::Reports::Security::Flag.new(type: flag['type'], origin: flag['origin'], description: flag['description'])
+          end
+
           def create_links(links)
             return [] unless links.is_a?(Array)
 
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index 3d92886cba8..143b930c669 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -12,7 +12,7 @@ module Gitlab
               end
 
               def initialize(report_type)
-                @report_type = report_type
+                @report_type = report_type.to_sym
               end
 
               delegate :validate, to: :schemer
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index d3bc3a38f1f..6feb693221b 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -5,9 +5,6 @@ module Gitlab
     module Pipeline
       module Chain
         class Build < Chain::Base
-          include Gitlab::Allowable
-          include Chain::Helpers
-
           def perform!
             @pipeline.assign_attributes(
               source: @command.source,
@@ -23,35 +20,12 @@ module Gitlab
               pipeline_schedule: @command.schedule,
               merge_request: @command.merge_request,
               external_pull_request: @command.external_pull_request,
-              locked: @command.project.default_pipeline_lock,
-              variables_attributes: variables_attributes
-            )
+              locked: @command.project.default_pipeline_lock)
           end
 
           def break?
             @pipeline.errors.any?
           end
-
-          private
-
-          def variables_attributes
-            variables = Array(@command.variables_attributes)
-
-            # We allow parent pipelines to pass variables to child pipelines since
-            # these variables are coming from internal configurations. We will check
-            # permissions to :set_pipeline_variables when those are injected upstream,
-            # to the parent pipeline.
-            # In other scenarios (e.g. multi-project pipelines or run pipeline via UI)
-            # the variables are provided from the outside and those should be guarded.
-            return variables if @command.creates_child_pipeline?
-
-            if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project)
-              error("Insufficient permissions to set pipeline variables")
-              variables = []
-            end
-
-            variables
-          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/build/associations.rb b/lib/gitlab/ci/pipeline/chain/build/associations.rb
index eb49c56bcd7..b5d63691849 100644
--- a/lib/gitlab/ci/pipeline/chain/build/associations.rb
+++ b/lib/gitlab/ci/pipeline/chain/build/associations.rb
@@ -6,7 +6,25 @@ module Gitlab
       module Chain
         class Build
           class Associations < Chain::Base
+            include Gitlab::Allowable
+            include Chain::Helpers
+
             def perform!
+              assign_pipeline_variables
+              assign_source_pipeline
+            end
+
+            def break?
+              @pipeline.errors.any?
+            end
+
+            private
+
+            def assign_pipeline_variables
+              @pipeline.variables_attributes = variables_attributes
+            end
+
+            def assign_source_pipeline
               return unless @command.bridge
 
               @pipeline.build_source_pipeline(
@@ -17,8 +35,45 @@ module Gitlab
               )
             end
 
-            def break?
-              false
+            def variables_attributes
+              variables = Array(@command.variables_attributes)
+              variables = apply_permissions(variables)
+              validate_uniqueness(variables)
+            end
+
+            def apply_permissions(variables)
+              # We allow parent pipelines to pass variables to child pipelines since
+              # these variables are coming from internal configurations. We will check
+              # permissions to :set_pipeline_variables when those are injected upstream,
+              # to the parent pipeline.
+              # In other scenarios (e.g. multi-project pipelines or run pipeline via UI)
+              # the variables are provided from the outside and those should be guarded.
+              return variables if @command.creates_child_pipeline?
+
+              if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project)
+                error("Insufficient permissions to set pipeline variables")
+                variables = []
+              end
+
+              variables
+            end
+
+            def validate_uniqueness(variables)
+              duplicated_keys = variables
+                .map { |var| var[:key] }
+                .tally
+                .filter_map { |key, count| key if count > 1 }
+
+              if duplicated_keys.empty?
+                variables
+              else
+                error(duplicate_variables_message(duplicated_keys), config_error: true)
+                []
+              end
+            end
+
+            def duplicate_variables_message(keys)
+              "Duplicate variable #{'name'.pluralize(keys.size)}: #{keys.join(', ')}"
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
index 1c0dfbdbee3..f637001f9f8 100644
--- a/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
+++ b/lib/gitlab/ci/pipeline/chain/cancel_pending_pipelines.rb
@@ -7,15 +7,19 @@ module Gitlab
         class CancelPendingPipelines < Chain::Base
           include Chain::Helpers
 
+          BATCH_SIZE = 25
+
+          # rubocop: disable CodeReuse/ActiveRecord
           def perform!
             return unless project.auto_cancel_pending_pipelines?
 
             Gitlab::OptimisticLocking.retry_lock(auto_cancelable_pipelines, name: 'cancel_pending_pipelines') do |cancelables|
-              cancelables.find_each do |cancelable|
-                cancelable.auto_cancel_running(pipeline)
+              cancelables.select(:id).each_batch(of: BATCH_SIZE) do |cancelables_batch|
+                auto_cancel_interruptible_pipelines(cancelables_batch.ids)
               end
             end
           end
+          # rubocop: enable CodeReuse/ActiveRecord
 
           def break?
             false
@@ -23,16 +27,21 @@ module Gitlab
 
           private
 
-          # rubocop: disable CodeReuse/ActiveRecord
           def auto_cancelable_pipelines
-            project.all_pipelines.ci_and_parent_sources
-              .where(ref: pipeline.ref)
-              .where.not(id: pipeline.same_family_pipeline_ids)
-              .where.not(sha: project.commit(pipeline.ref).try(:id))
+            project.all_pipelines.created_after(1.week.ago)
+              .ci_and_parent_sources
+              .for_ref(pipeline.ref)
+              .id_not_in(pipeline.same_family_pipeline_ids)
+              .where_not_sha(project.commit(pipeline.ref).try(:id))
               .alive_or_scheduled
+          end
+
+          def auto_cancel_interruptible_pipelines(pipeline_ids)
+            ::Ci::Pipeline
+              .id_in(pipeline_ids)
               .with_only_interruptible_builds
+              .each { |cancelable| cancelable.auto_cancel_running(pipeline) }
           end
-          # rubocop: enable CodeReuse/ActiveRecord
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 626eba97817..c9bc4ec411d 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -87,6 +87,13 @@ module Gitlab
             @metrics ||= ::Gitlab::Ci::Pipeline::Metrics
           end
 
+          def observe_step_duration(step_class, duration)
+            if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml)
+              metrics.pipeline_creation_step_duration_histogram
+                .observe({ step: step_class.name }, duration.seconds)
+            end
+          end
+
           def observe_creation_duration(duration)
             metrics.pipeline_creation_duration_histogram
               .observe({}, duration.seconds)
diff --git a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb
index 8a19e433483..092e7d43371 100644
--- a/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/content/external_project.rb
@@ -11,8 +11,12 @@ module Gitlab
                 strong_memoize(:content) do
                   next unless external_project_path?
 
-                  path_file, path_project = ci_config_path.split('@', 2)
-                  YAML.dump('include' => [{ 'project' => path_project, 'file' => path_file }])
+                  path_file, path_project, ref = extract_location_tokens
+
+                  config_location = { 'project' => path_project, 'file' => path_file }
+                  config_location['ref'] = ref if ref.present?
+
+                  YAML.dump('include' => [config_location])
                 end
               end
 
@@ -26,6 +30,18 @@ module Gitlab
               def external_project_path?
                 ci_config_path =~ /\A.+(yml|yaml)@.+\z/
               end
+
+              # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname
+              def extract_location_tokens
+                path_file, path_project = ci_config_path.split('@', 2)
+
+                if path_project.include? ":"
+                  project, ref = path_project.split(':', 2)
+                  [path_file, project, ref]
+                else
+                  [path_file, path_project]
+                end
+              end
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index bbfc6759b35..845eb6c7a42 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -14,9 +14,16 @@ module Gitlab
 
           def build!
             @sequence.each do |step_class|
+              step_start = ::Gitlab::Metrics::System.monotonic_time
               step = step_class.new(@pipeline, @command)
 
               step.perform!
+
+              @command.observe_step_duration(
+                step_class,
+                ::Gitlab::Metrics::System.monotonic_time - step_start
+              )
+
               break if step.break?
             end
 
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 10de77afe74..28df9f5386c 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -4,6 +4,8 @@ module Gitlab
   module Ci
     module Pipeline
       class Metrics
+        extend Gitlab::Utils::StrongMemoize
+
         def self.pipeline_creation_duration_histogram
           name = :gitlab_ci_pipeline_creation_duration_seconds
           comment = 'Pipeline creation duration'
@@ -13,6 +15,17 @@ module Gitlab
           ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
         end
 
+        def self.pipeline_creation_step_duration_histogram
+          strong_memoize(:pipeline_creation_step_histogram) do
+            name = :gitlab_ci_pipeline_creation_step_duration_seconds
+            comment = 'Duration of each pipeline creation step'
+            labels = { step: nil }
+            buckets = [0.01, 0.05, 0.1, 0.5, 1.0, 2.0, 5.0, 10.0, 15.0, 20.0, 50.0, 240.0]
+
+            ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
         def self.pipeline_security_orchestration_policy_processing_duration_histogram
           name = :gitlab_ci_pipeline_security_orchestration_policy_processing_duration_seconds
           comment = 'Pipeline security orchestration policy processing duration'
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index c393fed26de..934bf22d8ad 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -15,12 +15,7 @@ module Gitlab
             @context = context
             @pipeline = context.pipeline
             @seed_attributes = attributes
-            @stages_for_needs_lookup = if Feature.enabled?(:ci_same_stage_job_needs, @pipeline.project, default_enabled: :yaml)
-                                         (previous_stages + [current_stage]).compact
-                                       else
-                                         previous_stages
-                                       end
-
+            @stages_for_needs_lookup = (previous_stages + [current_stage]).compact
             @needs_attributes = dig(:needs_attributes)
             @resource_group_key = attributes.delete(:resource_group_key)
             @job_variables = @seed_attributes.delete(:job_variables)
@@ -123,6 +118,8 @@ module Gitlab
               return { environment: nil }
             end
 
+            build.persisted_environment = environment
+
             {
               deployment: Seed::Deployment.new(build, environment).to_resource,
               metadata_attributes: {
diff --git a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb
index f8ea6d4184c..a29fef6eb34 100644
--- a/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb
+++ b/lib/gitlab/ci/pipeline/seed/processable/resource_group.rb
@@ -28,7 +28,16 @@ module Gitlab
 
             def expanded_resource_group_key
               strong_memoize(:expanded_resource_group_key) do
-                ExpandVariables.expand(resource_group_key, -> { processable.simple_variables })
+                ExpandVariables.expand(resource_group_key, -> { variables })
+              end
+            end
+
+            def variables
+              processable.simple_variables.tap do |variables|
+                # Adding persisted environment variables
+                if processable.persisted_environment.present?
+                  variables.concat(processable.persisted_environment.predefined_variables)
+                end
               end
             end
           end
diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb
index 859aeb35f26..7f45d626922 100644
--- a/lib/gitlab/ci/queue/metrics.rb
+++ b/lib/gitlab/ci/queue/metrics.rb
@@ -97,7 +97,9 @@ module Gitlab
         def observe_queue_size(size_proc, runner_type)
           return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
-          self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f)
+          size = size_proc.call.to_f
+          self.class.queue_size_total.observe({ runner_type: runner_type }, size)
+          self.class.current_queue_size.set({ runner_type: runner_type }, size)
         end
 
         def observe_queue_time(metric, runner_type)
@@ -199,6 +201,15 @@ module Gitlab
           end
         end
 
+        def self.current_queue_size
+          strong_memoize(:current_queue_size) do
+            name = :gitlab_ci_current_queue_size
+            comment = 'Current size of initialized CI/CD builds queue'
+
+            Gitlab::Metrics.gauge(name, comment)
+          end
+        end
+
         def self.queue_iteration_duration_seconds
           strong_memoize(:queue_iteration_duration_seconds) do
             name = :gitlab_ci_queue_iteration_duration_seconds
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
index dc1c51b3ed0..39531e12f69 100644
--- a/lib/gitlab/ci/reports/security/finding.rb
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -10,6 +10,7 @@ module Gitlab
           attr_reader :compare_key
           attr_reader :confidence
           attr_reader :identifiers
+          attr_reader :flags
           attr_reader :links
           attr_reader :location
           attr_reader :metadata_version
@@ -30,10 +31,11 @@ module Gitlab
 
           delegate :file_path, :start_line, :end_line, to: :location
 
-          def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
+          def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
             @compare_key = compare_key
             @confidence = confidence
             @identifiers = identifiers
+            @flags = flags
             @links = links
             @location = location
             @metadata_version = metadata_version
@@ -58,6 +60,7 @@ module Gitlab
               compare_key
               confidence
               identifiers
+              flags
               links
               location
               metadata_version
diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb
new file mode 100644
index 00000000000..7e6cc758864
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/flag.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Reports
+      module Security
+        class Flag
+          attr_reader :type, :origin, :description
+
+          MAP = { 'flagged-as-likely-false-positive' => :false_positive }.freeze
+          DEFAULT_FLAG_TYPE = :false_positive
+
+          def flag_type
+            MAP.fetch(type, DEFAULT_FLAG_TYPE)
+          end
+
+          def initialize(type: nil, origin: nil, description: nil)
+            @type = type
+            @origin = origin
+            @description = description
+          end
+
+          def to_hash
+            {
+              flag_type: flag_type,
+              origin: origin,
+              description: description
+            }.compact
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index dbbb9a01dab..ee210e51232 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -32,7 +32,8 @@ module Gitlab
             user_blocked: 'pipeline user was blocked',
             ci_quota_exceeded: 'no more CI minutes available',
             no_matching_runner: 'no matching runner available',
-            trace_size_exceeded: 'log size limit exceeded'
+            trace_size_exceeded: 'log size limit exceeded',
+            builds_disabled: 'project builds are disabled'
           }.freeze
 
           private_constant :REASONS
diff --git a/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..9f0e9bcc1f2
--- /dev/null
+++ b/lib/gitlab/ci/templates/Android.latest.gitlab-ci.yml
@@ -0,0 +1,87 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Android.gitlab-ci.yml
+
+# Read more about this script on this blog post https://about.gitlab.com/2018/10/24/setting-up-gitlab-ci-for-android-projects/, by Jason Lenny
+# If you are interested in using Android with FastLane for publishing take a look at the Android-Fastlane template.
+
+image: openjdk:11-jdk
+
+variables:
+
+  # ANDROID_COMPILE_SDK is the version of Android you're compiling with.
+  # It should match compileSdkVersion.
+  ANDROID_COMPILE_SDK: "30"
+
+  # ANDROID_BUILD_TOOLS is the version of the Android build tools you are using.
+  # It should match buildToolsVersion.
+  ANDROID_BUILD_TOOLS: "30.0.3"
+
+  # It's what version of the command line tools we're going to download from the official site.
+  # Official Site-> https://developer.android.com/studio/index.html
+  # There, look down below at the cli tools only, sdk tools package is of format:
+  #        commandlinetools-os_type-ANDROID_SDK_TOOLS_latest.zip
+  # when the script was last modified for latest compileSdkVersion, it was which is written down below
+  ANDROID_SDK_TOOLS: "7583922"
+
+# Packages installation before running script
+before_script:
+  - apt-get --quiet update --yes
+  - apt-get --quiet install --yes wget tar unzip lib32stdc++6 lib32z1
+
+  # Setup path as ANDROID_SDK_ROOT for moving/exporting the downloaded sdk into it
+  - export ANDROID_SDK_ROOT="${PWD}/android-home"
+  # Create a new directory at specified location
+  - install -d $ANDROID_SDK_ROOT
+  # Here we are installing androidSDK tools from official source,
+  # (the key thing here is the url from where you are downloading these sdk tool for command line, so please do note this url pattern there and here as well)
+  # after that unzipping those tools and
+  # then running a series of SDK manager commands to install necessary android SDK packages that'll allow the app to build
+  - wget --output-document=$ANDROID_SDK_ROOT/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-${ANDROID_SDK_TOOLS}_latest.zip
+  # move to the archive at ANDROID_SDK_ROOT
+  - pushd $ANDROID_SDK_ROOT
+  - unzip -d cmdline-tools cmdline-tools.zip
+  - pushd cmdline-tools
+  # since commandline tools version 7583922 the root folder is named "cmdline-tools" so we rename it if necessary
+  - mv cmdline-tools tools || true
+  - popd
+  - popd
+  - export PATH=$PATH:${ANDROID_SDK_ROOT}/cmdline-tools/tools/bin/
+
+  # Nothing fancy here, just checking sdkManager version
+  - sdkmanager --version
+
+  # use yes to accept all licenses
+  - yes | sdkmanager --licenses || true
+  - sdkmanager "platforms;android-${ANDROID_COMPILE_SDK}"
+  - sdkmanager "platform-tools"
+  - sdkmanager "build-tools;${ANDROID_BUILD_TOOLS}"
+
+  # Not necessary, but just for surity
+  - chmod +x ./gradlew
+
+# Basic android and gradle stuff
+# Check linting
+lintDebug:
+  interruptible: true
+  stage: build
+  script:
+    - ./gradlew -Pci --console=plain :app:lintDebug -PbuildDir=lint
+
+# Make Project
+assembleDebug:
+  interruptible: true
+  stage: build
+  script:
+    - ./gradlew assembleDebug
+  artifacts:
+    paths:
+      - app/build/outputs/
+
+# Run all tests, if any fails, interrupt the pipeline(fail it)
+debugTests:
+  interruptible: true
+  stage: test
+  script:
+    - ./gradlew -Pci --console=plain :app:testDebug
diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
index cf99d722e4d..5efa557d7eb 100644
--- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml
@@ -1,6 +1,9 @@
+variables:
+  AUTO_BUILD_IMAGE_VERSION: 'v1.0.0'
+
 build:
   stage: build
-  image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:v1.0.0'
+  image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}'
   variables:
     DOCKER_TLS_CERTDIR: ''
   services:
diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..6a3b0cfa9e7
--- /dev/null
+++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml
@@ -0,0 +1,41 @@
+# WARNING: This latest template is for internal FEATURE-FLAG TESTING ONLY.
+#          It is not meant to be used with `include:`.
+#          This template is scheduled for removal when testing is complete: https://gitlab.com/gitlab-org/gitlab/-/issues/337987
+
+variables:
+  AUTO_BUILD_IMAGE_VERSION: 'v1.3.1'
+
+build:
+  stage: build
+  image: 'registry.gitlab.com/gitlab-org/cluster-integration/auto-build-image:${AUTO_BUILD_IMAGE_VERSION}'
+  variables:
+    DOCKER_TLS_CERTDIR: ''
+  services:
+    - name: 'docker:20.10.6-dind'
+      command: ['--tls=false', '--host=tcp://0.0.0.0:2375']
+  script:
+    - |
+      if [[ -z "$CI_COMMIT_TAG" ]]; then
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_SHA}
+      else
+        export CI_APPLICATION_REPOSITORY=${CI_APPLICATION_REPOSITORY:-$CI_REGISTRY_IMAGE}
+        export CI_APPLICATION_TAG=${CI_APPLICATION_TAG:-$CI_COMMIT_TAG}
+      fi
+    - /build/build.sh
+  rules:
+    - if: '$BUILD_DISABLED'
+      when: never
+    - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"'
+      when: never
+    - if: '$CI_COMMIT_TAG || $CI_COMMIT_BRANCH'
+
+build_artifact:
+  stage: build
+  script:
+    - printf "To build your project, please create a build_artifact job into your .gitlab-ci.yml file.\nMore information at https://docs.gitlab.com/ee/ci/cloud_deployment\n"
+    - exit 1
+  rules:
+    - if: '$BUILD_DISABLED'
+      when: never
+    - if: '$AUTO_DEVOPS_PLATFORM_TARGET == "EC2"'
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 43ecc4b96d5..00b771f1e5c 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,7 @@ code_quality:
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24"
+    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1"
   needs: []
   script:
     - export SOURCE_CODE=$PWD
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 208951fa1a1..e0627b85aba 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,8 @@
+variables:
+  DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0'
+
 .dast-auto-deploy:
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}"
 
 dast_environment_deploy:
   extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 5c466f0984c..2df985cfbb5 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,8 @@
+variables:
+  AUTO_DEPLOY_IMAGE_VERSION: 'v2.12.0'
+
 .auto-deploy:
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}"
   dependencies: []
 
 review:
@@ -96,6 +99,8 @@ canary:
     name: production
     url: http://$CI_PROJECT_PATH_SLUG.$KUBE_INGRESS_BASE_DOMAIN
   rules:
+    - if: '$CI_DEPLOY_FREEZE != null'
+      when: never
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
     - if: '$CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH'
@@ -125,6 +130,8 @@ canary:
 production:
   <<: *production_template
   rules:
+    - if: '$CI_DEPLOY_FREEZE != null'
+      when: never
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
     - if: '$STAGING_ENABLED'
@@ -141,6 +148,8 @@ production_manual:
   <<: *production_template
   allow_failure: false
   rules:
+    - if: '$CI_DEPLOY_FREEZE != null'
+      when: never
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_ENABLED'
@@ -177,6 +186,8 @@ production_manual:
   resource_group: production
   allow_failure: true
   rules:
+    - if: '$CI_DEPLOY_FREEZE != null'
+      when: never
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "timed"'
@@ -190,6 +201,8 @@ production_manual:
 .timed_rollout_template: &timed_rollout_template
   <<: *rollout_template
   rules:
+    - if: '$CI_DEPLOY_FREEZE != null'
+      when: never
     - if: '$CI_KUBERNETES_ACTIVE == null || $CI_KUBERNETES_ACTIVE == ""'
       when: never
     - if: '$INCREMENTAL_ROLLOUT_MODE == "manual"'
diff --git a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
index a130b09c51a..1ec1aa60d88 100644
--- a/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Helm-2to3.gitlab-ci.yml
@@ -41,9 +41,9 @@
         echo "Adopting Helm v2 manifests from $release"
         # some resource kinds must be listed explicitly https://github.com/kubernetes/kubernetes/issues/42885
         for name in $(kubectl -n "$KUBE_NAMESPACE" get all,ingress,daemonset -o name -l chart="$chart"); do
-          kubectl annotate --overwrite "$name" meta.helm.sh/release-name="$release"
-          kubectl annotate --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE"
-          kubectl label --overwrite "$name" app.kubernetes.io/managed-by=Helm
+          kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-name="$release"
+          kubectl annotate -n "$KUBE_NAMESPACE" --overwrite "$name" meta.helm.sh/release-namespace="$KUBE_NAMESPACE"
+          kubectl label -n "$KUBE_NAMESPACE" --overwrite "$name" app.kubernetes.io/managed-by=Helm
         done
       done
     # migrate each release
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index c458ab6a00a..081a3a6cc78 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -16,6 +16,9 @@ stages:
 init:
   extends: .terraform:init
 
+fmt:
+  extends: .terraform:fmt
+
 validate:
   extends: .terraform:validate
 
diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
index 39c3374e534..e696c75253e 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
@@ -20,7 +20,6 @@ cache:
   key: "${TF_ROOT}"
   paths:
     - ${TF_ROOT}/.terraform/
-    - ${TF_ROOT}/.terraform.lock.hcl
 
 .init: &init
   stage: init
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index c30860ad174..3a70e6bc4b8 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -20,7 +20,6 @@ cache:
   key: "${TF_ROOT}"
   paths:
     - ${TF_ROOT}/.terraform/
-    - ${TF_ROOT}/.terraform.lock.hcl
 
 .terraform:init: &terraform_init
   stage: init
@@ -28,6 +27,14 @@ cache:
     - cd ${TF_ROOT}
     - gitlab-terraform init
 
+.terraform:fmt: &terraform_fmt
+  stage: validate
+  needs: []
+  script:
+    - cd ${TF_ROOT}
+    - gitlab-terraform fmt -check -recursive
+  allow_failure: true
+
 .terraform:validate: &terraform_validate
   stage: validate
   script:
diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
index dd88953b9a4..841f17767eb 100644
--- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml
@@ -21,7 +21,7 @@
 #
 # The deploy stage copies the exe and msi from build stage to a network drive
 # You need to have the network drive mapped as Local System user for gitlab-runner service to see it
-# The best way to persist the mapping is via a scheduled task (see: https://stackoverflow.com/a/7867064/1288473),
+# The best way to persist the mapping is via a scheduled task
 # running the following batch command: net use P: \\x.x.x.x\Projects /u:your_user your_pass /persistent:yes
 
 # place project specific paths in variables to make the rest of the script more generic
diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb
index f9798023838..72a94dcd412 100644
--- a/lib/gitlab/ci/trace.rb
+++ b/lib/gitlab/ci/trace.rb
@@ -4,6 +4,7 @@ module Gitlab
   module Ci
     class Trace
       include ::Gitlab::ExclusiveLeaseHelpers
+      include ::Gitlab::Utils::StrongMemoize
       include Checksummable
 
       LOCK_TTL = 10.minutes
@@ -23,6 +24,8 @@ module Gitlab
       attr_reader :job
 
       delegate :old_trace, to: :job
+      delegate :can_attempt_archival_now?, :increment_archival_attempts!,
+        :archival_attempts_message, to: :trace_metadata
 
       def initialize(job)
         @job = job
@@ -188,11 +191,7 @@ module Gitlab
       def unsafe_archive!
         raise ArchiveError, 'Job is not finished yet' unless job.complete?
 
-        if trace_artifact
-          unsafe_trace_cleanup!
-
-          raise AlreadyArchivedError, 'Could not archive again'
-        end
+        unsafe_trace_conditionally_cleanup_before_retry!
 
         if job.trace_chunks.any?
           Gitlab::Ci::Trace::ChunkedIO.new(job) do |stream|
@@ -212,12 +211,19 @@ module Gitlab
         end
       end
 
-      def unsafe_trace_cleanup!
+      def already_archived?
+        # TODO check checksum to ensure archive completed successfully
+        # See https://gitlab.com/gitlab-org/gitlab/-/issues/259619
+        trace_artifact.archived_trace_exists?
+      end
+
+      def unsafe_trace_conditionally_cleanup_before_retry!
         return unless trace_artifact
 
-        if trace_artifact.archived_trace_exists?
+        if already_archived?
           # An archive already exists, so make sure to remove the trace chunks
           erase_trace_chunks!
+          raise AlreadyArchivedError, 'Could not archive again'
         else
           # An archive already exists, but its associated file does not, so remove it
           trace_artifact.destroy!
@@ -251,11 +257,19 @@ module Gitlab
         File.open(path) do |stream|
           # TODO: Set `file_format: :raw` after we've cleaned up legacy traces migration
           # https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/20307
-          job.create_job_artifacts_trace!(
+          trace_artifact = job.create_job_artifacts_trace!(
             project: job.project,
             file_type: :trace,
             file: stream,
             file_sha256: self.class.hexdigest(path))
+
+          trace_metadata.track_archival!(trace_artifact.id)
+        end
+      end
+
+      def trace_metadata
+        strong_memoize(:trace_metadata) do
+          job.ensure_trace_metadata!
         end
       end
 
diff --git a/lib/gitlab/ci/trace/backoff.rb b/lib/gitlab/ci/trace/backoff.rb
new file mode 100644
index 00000000000..c13d88cced1
--- /dev/null
+++ b/lib/gitlab/ci/trace/backoff.rb
@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class Trace
+      ##
+      # Trace::Backoff class is responsible for calculating a backoff value
+      # for when to be able to retry archiving a build's trace
+      #
+      # Because we're updating `last_archival_attempt_at` timestamp with every
+      # failed archival attempt, we need to be sure that sum of the backoff values
+      # for 1..MAX_ATTEMPTS is under 7 days(CHUNK_REDIS_TTL).
+      #
+      class Backoff
+        include Gitlab::Utils::StrongMemoize
+
+        MAX_JITTER_VALUE = 4
+
+        attr_reader :archival_attempts
+
+        def initialize(archival_attempts)
+          @archival_attempts = archival_attempts
+        end
+
+        def value
+          (((chunks_ttl / (3.5 * max_attempts)) * archival_attempts) / 1.hour).hours
+        end
+
+        # This formula generates an increasing delay between executions
+        # 9.6, 19.2, 28.8, 38.4, 48.0 + a random amount of time to
+        # change the order of execution for the jobs.
+        # With maximum value for each call to rand(4), this sums up to 6.8 days
+        # and with minimum values is 6 days.
+        #
+        def value_with_jitter
+          value + jitter
+        end
+
+        private
+
+        def jitter
+          rand(MAX_JITTER_VALUE).hours
+        end
+
+        def chunks_ttl
+          ::Ci::BuildTraceChunks::RedisBase::CHUNK_REDIS_TTL
+        end
+
+        def max_attempts
+          ::Ci::BuildTraceMetadata::MAX_ATTEMPTS
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index fdc598c025a..2d31049a0c9 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -3,7 +3,6 @@
 module Gitlab
   module Ci
     class Trace
-      # This was inspired from: http://stackoverflow.com/a/10219411/1520132
       class Stream
         BUFFER_SIZE = 4096
         LIMIT_SIZE = 500.kilobytes
diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb
index ef9ba1b73c7..09c75a2b3f1 100644
--- a/lib/gitlab/ci/variables/collection.rb
+++ b/lib/gitlab/ci/variables/collection.rb
@@ -10,7 +10,7 @@ module Gitlab
 
         def initialize(variables = [], errors = nil)
           @variables = []
-          @variables_by_key = {}
+          @variables_by_key = Hash.new { |h, k| h[k] = [] }
           @errors = errors
 
           variables.each { |variable| self.append(variable) }
@@ -19,7 +19,7 @@ module Gitlab
         def append(resource)
           item = Collection::Item.fabricate(resource)
           @variables.append(item)
-          @variables_by_key[item[:key]] = item
+          @variables_by_key[item[:key]] << item
 
           self
         end
@@ -46,7 +46,12 @@ module Gitlab
         end
 
         def [](key)
-          @variables_by_key[key]
+          all(key)&.last
+        end
+
+        def all(key)
+          vars = @variables_by_key[key]
+          vars unless vars.empty?
         end
 
         def size
@@ -72,7 +77,7 @@ module Gitlab
             match = Regexp.last_match
             if match[:key]
               # we matched variable
-              if variable = @variables_by_key[match[:key]]
+              if variable = self[match[:key]]
                 variable.value
               elsif keep_undefined
                 match[0]
@@ -85,7 +90,7 @@ module Gitlab
         end
 
         def sort_and_expand_all(project, keep_undefined: false)
-          return self if Feature.disabled?(:variable_inside_variable, project)
+          return self if Feature.disabled?(:variable_inside_variable, project, default_enabled: :yaml)
 
           sorted = Sort.new(self)
           return self.class.new(self, sorted.errors) unless sorted.valid?
diff --git a/lib/gitlab/ci/variables/collection/sort.rb b/lib/gitlab/ci/variables/collection/sort.rb
index 90a929b8a07..62637825c15 100644
--- a/lib/gitlab/ci/variables/collection/sort.rb
+++ b/lib/gitlab/ci/variables/collection/sort.rb
@@ -42,7 +42,7 @@ module Gitlab
             depends_on = var_item.depends_on
             return unless depends_on
 
-            depends_on.filter_map { |var_ref_name| @collection[var_ref_name] }.each(&block)
+            depends_on.filter_map { |var_ref_name| @collection.all(var_ref_name) }.flatten.each(&block)
           end
         end
       end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index c94fa84f608..1aa3dbc5e47 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -47,9 +47,7 @@ module Gitlab
           validate_job!(name, job)
         end
 
-        if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml)
-          YamlProcessor::Dag.check_circular_dependencies!(@jobs)
-        end
+        YamlProcessor::Dag.check_circular_dependencies!(@jobs)
       end
 
       def validate_job!(name, job)
@@ -103,16 +101,8 @@ module Gitlab
         job_stage_index = stage_index(name)
         dependency_stage_index = stage_index(dependency)
 
-        if ::Feature.enabled?(:ci_same_stage_job_needs, @opts[:project], default_enabled: :yaml)
-          unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index
-            error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages")
-          end
-        else
-          # A dependency might be defined later in the configuration
-          # with a stage that does not exist
-          unless dependency_stage_index.present? && dependency_stage_index < job_stage_index
-            error!("#{name} job: #{dependency_type} #{dependency} is not defined in prior stages")
-          end
+        unless dependency_stage_index.present? && dependency_stage_index <= job_stage_index
+          error!("#{name} job: #{dependency_type} #{dependency} is not defined in current or prior stages")
         end
       end