Add latest changes from gitlab-org/gitlab@14-7-stable-eev14.7.0-rc42

25 files changed, 332 insertions, 84 deletions

diff --git a/lib/gitlab/ci/build/policy/refs.rb b/lib/gitlab/ci/build/policy/refs.rb
index afe0ccb361e..7ade9ca5085 100644
--- a/lib/gitlab/ci/build/policy/refs.rb
+++ b/lib/gitlab/ci/build/policy/refs.rb
@@ -35,7 +35,10 @@ module Gitlab
             # patterns can be matched only when branch or tag is used
             # the pattern matching does not work for merge requests pipelines
             if pipeline.branch? || pipeline.tag?
-              if regexp = Gitlab::UntrustedRegexp::RubySyntax.fabricate(pattern, fallback: true)
+              regexp = Gitlab::UntrustedRegexp::RubySyntax
+                .fabricate(pattern, fallback: true, project: pipeline.project)
+
+              if regexp
                 regexp.match?(pipeline.ref)
               else
                 pattern == pipeline.ref
diff --git a/lib/gitlab/ci/build/status/reason.rb b/lib/gitlab/ci/build/status/reason.rb
new file mode 100644
index 00000000000..82e07faef63
--- /dev/null
+++ b/lib/gitlab/ci/build/status/reason.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Build
+      module Status
+        class Reason
+          attr_reader :build, :failure_reason, :exit_code
+
+          def initialize(build, failure_reason, exit_code = nil)
+            @build = build
+            @failure_reason = failure_reason
+            @exit_code = exit_code
+          end
+
+          def failure_reason_enum
+            ::CommitStatus.failure_reasons[failure_reason]
+          end
+
+          def force_allow_failure?
+            return false if exit_code.nil?
+
+            !build.allow_failure? && build.allowed_to_fail_with_code?(exit_code)
+          end
+
+          def self.fabricate(build, reason)
+            if reason.is_a?(self)
+              new(build, reason.failure_reason, reason.exit_code)
+            else
+              new(build, reason)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 42b487fdf81..4c98941e032 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -36,7 +36,7 @@ module Gitlab
         end
 
         @root = self.logger.instrument(:config_compose) do
-          Entry::Root.new(@config).tap(&:compose!)
+          Entry::Root.new(@config, project: project, user: user).tap(&:compose!)
         end
       rescue *rescue_errors => e
         raise Config::ConfigError, e.message
diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb
index e6290ef2479..41a3c87037b 100644
--- a/lib/gitlab/ci/config/entry/root.rb
+++ b/lib/gitlab/ci/config/entry/root.rb
@@ -59,7 +59,8 @@ module Gitlab
 
           entry :types, Entry::Stages,
             description: 'Deprecated: stages for this pipeline.',
-            reserved: true
+            reserved: true,
+            deprecation: { deprecated: '9.0', warning: '14.8', removed: '15.0', documentation: 'https://docs.gitlab.com/ee/ci/yaml/#deprecated-keywords' }
 
           entry :cache, Entry::Caches,
             description: 'Configure caching between build jobs.',
@@ -122,8 +123,9 @@ module Gitlab
             # Deprecated `:types` key workaround - if types are defined and
             # stages are not defined we use types definition as stages.
             #
-            if types_defined? && !stages_defined?
-              @entries[:stages] = @entries[:types]
+            if types_defined?
+              @entries[:stages] = @entries[:types] unless stages_defined?
+              log_and_warn_deprecated_entry(@entries[:types])
             end
 
             @entries.delete(:types)
diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb
new file mode 100644
index 00000000000..278353220e4
--- /dev/null
+++ b/lib/gitlab/ci/jwt_v2.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    class JwtV2 < Jwt
+      private
+
+      def reserved_claims
+        super.merge(
+          iss: Settings.gitlab.base_url,
+          sub:  "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}",
+          aud: Settings.gitlab.base_url
+        )
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 15b0ff3c04d..54b54bd0514 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -9,13 +9,13 @@ module Gitlab
           include Gitlab::Utils::StrongMemoize
 
           def perform!
-            logger.instrument(:pipeline_save) do
+            logger.instrument_with_sql(:pipeline_save) do
               BulkInsertableAssociations.with_bulk_insert do
-                tags = extract_tag_list_by_status
-
-                pipeline.transaction do
-                  pipeline.save!
-                  CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags?
+                with_bulk_insert_tags do
+                  pipeline.transaction do
+                    pipeline.save!
+                    CommitStatus.bulk_insert_tags!(statuses) if bulk_insert_tags?
+                  end
                 end
               end
             end
@@ -29,32 +29,26 @@ module Gitlab
 
           private
 
-          def statuses
-            strong_memoize(:statuses) do
-              pipeline.stages.flat_map(&:statuses)
+          def bulk_insert_tags?
+            strong_memoize(:bulk_insert_tags) do
+              ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml)
             end
           end
 
-          # We call `job.tag_list=` to assign tags to the jobs from the
-          # Chain::Seed step which uses the `@tag_list` instance variable to
-          # store them on the record. We remove them here because we want to
-          # bulk insert them, otherwise they would be inserted and assigned one
-          # by one with callbacks. We must use `remove_instance_variable`
-          # because having the instance variable defined would still run the callbacks
-          def extract_tag_list_by_status
-            return {} unless bulk_insert_tags?
-
-            statuses.each.with_object({}) do |job, acc|
-              tag_list = job.clear_memoization(:tag_list)
-              next unless tag_list
-
-              acc[job.name] = tag_list
-            end
+          def with_bulk_insert_tags
+            previous = Thread.current['ci_bulk_insert_tags']
+            Thread.current['ci_bulk_insert_tags'] = bulk_insert_tags?
+            yield
+          ensure
+            Thread.current['ci_bulk_insert_tags'] = previous
           end
 
-          def bulk_insert_tags?
-            strong_memoize(:bulk_insert_tags) do
-              ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml)
+          def statuses
+            strong_memoize(:statuses) do
+              pipeline
+                .stages
+                .flat_map(&:statuses)
+                .select { |status| status.respond_to?(:tag_list) }
             end
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb
index b92aa89d62d..b913ba3c87d 100644
--- a/lib/gitlab/ci/pipeline/chain/create_deployments.rb
+++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb
@@ -5,8 +5,6 @@ module Gitlab
     module Pipeline
       module Chain
         class CreateDeployments < Chain::Base
-          DeploymentCreationError = Class.new(StandardError)
-
           def perform!
             return unless pipeline.create_deployment_in_separate_transaction?
 
@@ -24,18 +22,7 @@ module Gitlab
           end
 
           def create_deployment(build)
-            return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present?
-
-            deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment
-              .new(build, build.persisted_environment).to_resource
-
-            return unless deployment
-
-            deployment.deployable = build
-            deployment.save!
-          rescue ActiveRecord::RecordInvalid => e
-            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
-              DeploymentCreationError.new(e.message), build_id: build.id)
+            ::Deployments::CreateForBuildService.new.execute(build)
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index 356eeb76908..feae123f216 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -53,13 +53,18 @@ module Gitlab
           end
 
           def context
-            Gitlab::Ci::Pipeline::Seed::Context.new(pipeline, root_variables: root_variables)
+            Gitlab::Ci::Pipeline::Seed::Context.new(
+              pipeline,
+              root_variables: root_variables,
+              logger: logger
+            )
           end
 
           def root_variables
             logger.instrument(:pipeline_seed_merge_variables) do
               ::Gitlab::Ci::Variables::Helpers.merge_variables(
-                @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables
+                @command.yaml_processor_result.root_variables,
+                @command.workflow_rules_result.variables
               )
             end
           end
diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb
index 97f7dddd09a..fbba12c11a9 100644
--- a/lib/gitlab/ci/pipeline/logger.rb
+++ b/lib/gitlab/ci/pipeline/logger.rb
@@ -37,6 +37,16 @@ module Gitlab
           result
         end
 
+        def instrument_with_sql(operation, &block)
+          op_start_db_counters = current_db_counter_payload
+
+          result = instrument(operation, &block)
+
+          observe_sql_counters(operation, op_start_db_counters, current_db_counter_payload)
+
+          result
+        end
+
         def observe(operation, value)
           return unless enabled?
 
@@ -50,11 +60,20 @@ module Gitlab
             class: self.class.name.to_s,
             pipeline_creation_caller: caller,
             project_id: project.id,
-            pipeline_id: pipeline.id,
             pipeline_persisted: pipeline.persisted?,
             pipeline_source: pipeline.source,
             pipeline_creation_service_duration_s: age
-          }.stringify_keys.merge(observations_hash)
+          }
+
+          if pipeline.persisted?
+            attributes[:pipeline_builds_tags_count] = pipeline.tags_count
+            attributes[:pipeline_builds_distinct_tags_count] = pipeline.distinct_tags_count
+            attributes[:pipeline_id] = pipeline.id
+          end
+
+          attributes.compact!
+          attributes.stringify_keys!
+          attributes.merge!(observations_hash)
 
           destination.info(attributes)
         end
@@ -97,6 +116,19 @@ module Gitlab
         def observations
           @observations ||= Hash.new { |hash, key| hash[key] = [] }
         end
+
+        def observe_sql_counters(operation, start_db_counters, end_db_counters)
+          end_db_counters.each do |key, value|
+            result = value - start_db_counters.fetch(key, 0)
+            next if result == 0
+
+            observe("#{operation}_#{key}", result)
+          end
+        end
+
+        def current_db_counter_payload
+          ::Gitlab::Metrics::Subscribers::ActiveRecord.db_counter_payload
+        end
       end
     end
   end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 762292f0fa3..5a0ad695741 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -41,12 +41,14 @@ module Gitlab
 
           def included?
             strong_memoize(:inclusion) do
-              if @using_rules
-                rules_result.pass?
-              elsif @using_only || @using_except
-                all_of_only? && none_of_except?
-              else
-                true
+              logger.instrument(:pipeline_seed_build_inclusion) do
+                if @using_rules
+                  rules_result.pass?
+                elsif @using_only || @using_except
+                  all_of_only? && none_of_except?
+                else
+                  true
+                end
               end
             end
           end
@@ -122,6 +124,8 @@ module Gitlab
 
           private
 
+          delegate :logger, to: :@context
+
           def all_of_only?
             @only.all? { |spec| spec.satisfied_by?(@pipeline, evaluate_context) }
           end
diff --git a/lib/gitlab/ci/pipeline/seed/context.rb b/lib/gitlab/ci/pipeline/seed/context.rb
index 6194a78f682..c0b8ebeb833 100644
--- a/lib/gitlab/ci/pipeline/seed/context.rb
+++ b/lib/gitlab/ci/pipeline/seed/context.rb
@@ -5,11 +5,18 @@ module Gitlab
     module Pipeline
       module Seed
         class Context
-          attr_reader :pipeline, :root_variables
+          attr_reader :pipeline, :root_variables, :logger
 
-          def initialize(pipeline, root_variables: [])
+          def initialize(pipeline, root_variables: [], logger: nil)
             @pipeline = pipeline
             @root_variables = root_variables
+            @logger = logger || build_logger
+          end
+
+          private
+
+          def build_logger
+            ::Gitlab::Ci::Pipeline::Logger.new(project: pipeline.project)
           end
         end
       end
diff --git a/lib/gitlab/ci/queue/metrics.rb b/lib/gitlab/ci/queue/metrics.rb
index 7f45d626922..54fb1d19ea8 100644
--- a/lib/gitlab/ci/queue/metrics.rb
+++ b/lib/gitlab/ci/queue/metrics.rb
@@ -69,17 +69,6 @@ module Gitlab
           self.class.attempt_counter.increment
         end
 
-        # rubocop: disable CodeReuse/ActiveRecord
-        def jobs_running_for_project(job)
-          return '+Inf' unless runner.instance_type?
-
-          # excluding currently started job
-          running_jobs_count = job.project.builds.running.where(runner: ::Ci::Runner.instance_type)
-                                  .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1
-          running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET ? running_jobs_count : "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+"
-        end
-        # rubocop: enable CodeReuse/ActiveRecord
-
         def increment_queue_operation(operation)
           self.class.increment_queue_operation(operation)
         end
@@ -242,6 +231,32 @@ module Gitlab
             Gitlab::Metrics.histogram(name, comment, labels, buckets)
           end
         end
+
+        private
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def jobs_running_for_project(job)
+          return '+Inf' unless runner.instance_type?
+
+          # excluding currently started job
+          running_jobs_count = running_jobs_relation(job)
+            .limit(JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET + 1).count - 1
+
+          if running_jobs_count < JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET
+            running_jobs_count
+          else
+            "#{JOBS_RUNNING_FOR_PROJECT_MAX_BUCKET}+"
+          end
+        end
+
+        def running_jobs_relation(job)
+          if ::Feature.enabled?(:ci_pending_builds_maintain_denormalized_data, default_enabled: :yaml)
+            ::Ci::RunningBuild.instance_type.where(project_id: job.project_id)
+          else
+            job.project.builds.running.where(runner: ::Ci::Runner.instance_type)
+          end
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
       end
     end
   end
diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb
index 7e5afbad806..a4434e2c144 100644
--- a/lib/gitlab/ci/status/build/factory.rb
+++ b/lib/gitlab/ci/status/build/factory.rb
@@ -14,7 +14,8 @@ module Gitlab
               Status::Build::WaitingForResource,
               Status::Build::Preparing,
               Status::Build::Pending,
-              Status::Build::Skipped],
+              Status::Build::Skipped,
+              Status::Build::WaitingForApproval],
              [Status::Build::Cancelable,
               Status::Build::Retryable],
              [Status::Build::FailedUnmetPrerequisites,
diff --git a/lib/gitlab/ci/status/build/waiting_for_approval.rb b/lib/gitlab/ci/status/build/waiting_for_approval.rb
new file mode 100644
index 00000000000..59869a947a9
--- /dev/null
+++ b/lib/gitlab/ci/status/build/waiting_for_approval.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Status
+      module Build
+        class WaitingForApproval < Status::Extended
+          def illustration
+            {
+              image: 'illustrations/manual_action.svg',
+              size: 'svg-394',
+              title: 'Waiting for approval',
+              content: "This job deploys to the protected environment \"#{subject.deployment&.environment&.name}\" which requires approvals. Use the Deployments API to approve or reject the deployment."
+            }
+          end
+
+          def self.matches?(build, user)
+            build.waiting_for_deployment_approval?
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb
index a299df7e2d9..29f3731a9b4 100644
--- a/lib/gitlab/ci/tags/bulk_insert.rb
+++ b/lib/gitlab/ci/tags/bulk_insert.rb
@@ -4,12 +4,13 @@ module Gitlab
   module Ci
     module Tags
       class BulkInsert
+        include Gitlab::Utils::StrongMemoize
+
         TAGGINGS_BATCH_SIZE = 1000
         TAGS_BATCH_SIZE = 500
 
-        def initialize(statuses, tag_list_by_status)
+        def initialize(statuses)
           @statuses = statuses
-          @tag_list_by_status = tag_list_by_status
         end
 
         def insert!
@@ -20,7 +21,18 @@ module Gitlab
 
         private
 
-        attr_reader :statuses, :tag_list_by_status
+        attr_reader :statuses
+
+        def tag_list_by_status
+          strong_memoize(:tag_list_by_status) do
+            statuses.each.with_object({}) do |status, acc|
+              tag_list = status.tag_list
+              next unless tag_list
+
+              acc[status] = tag_list
+            end
+          end
+        end
 
         def persist_build_tags!
           all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?)
@@ -54,7 +66,7 @@ module Gitlab
 
         def build_taggings_attributes(tag_records_by_name)
           taggings = statuses.flat_map do |status|
-            tag_list = tag_list_by_status[status.name]
+            tag_list = tag_list_by_status[status]
             next unless tag_list
 
             tags = tag_records_by_name.values_at(*tag_list)
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 00b771f1e5c..6942631a97f 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,7 @@ code_quality:
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.24-gitlab.1"
+    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.26"
   needs: []
   script:
     - export SOURCE_CODE=$PWD
diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
index 18f0f20203d..42487cc0c67 100644
--- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
@@ -14,6 +14,8 @@ variables:
   image: "$SECURE_ANALYZERS_PREFIX/secrets:$SECRETS_ANALYZER_VERSION"
   services: []
   allow_failure: true
+  variables:
+    GIT_DEPTH: "50"
   # `rules` must be overridden explicitly by each child job
   # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444
   artifacts:
@@ -29,8 +31,16 @@ secret_detection:
   script:
     - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
     - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi
-    - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
-    - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
-    - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt
+    - |
+      git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
+      git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/${CI_DEFAULT_BRANCH}..refs/remotes/origin/${CI_COMMIT_REF_NAME} >${CI_COMMIT_SHA}_commit_list.txt
+      if [[ $(wc -l <${CI_COMMIT_SHA}_commit_list.txt) -eq "0" ]]; then
+        # if git log produces 0 or 1 commits we should scan $CI_COMMIT_SHA only
+        export SECRET_DETECTION_COMMITS=$CI_COMMIT_SHA
+      else
+        # +1 because busybox wc only counts \n and there is no trailing \n
+        echo "scanning $(($(wc -l <${CI_COMMIT_SHA}_commit_list.txt) + 1)) commits"
+        export SECRET_DETECTION_COMMITS_FILE=${CI_COMMIT_SHA}_commit_list.txt
+      fi
     - /analyzer run
     - rm "$CI_COMMIT_SHA"_commit_list.txt
diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
index 0c8b98dc1cf..1660a9250e3 100644
--- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
@@ -29,7 +29,7 @@ before_script:
   - ruby -v  # Print out ruby version for debugging
   # Uncomment next line if your rails app needs a JS runtime:
   # - apt-get update -q && apt-get install nodejs -yqq
-  - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby
+  - bundle config set path 'vendor'  # Install dependencies into ./vendor/ruby
   - bundle install -j $(nproc)
 
 # Optional - Delete if not using `rubocop`
diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
index 7243f240eed..f7f016b5e57 100644
--- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml
@@ -33,6 +33,7 @@ coverage_fuzzing_unlicensed:
   before_script:
     - export COVFUZZ_JOB_TOKEN=$CI_JOB_TOKEN
     - export COVFUZZ_PRIVATE_TOKEN=$CI_PRIVATE_TOKEN
+    - export COVFUZZ_PROJECT_PATH=$CI_PROJECT_PATH
     - export COVFUZZ_PROJECT_ID=$CI_PROJECT_ID
     - if [ -x "$(command -v apt-get)" ] ; then apt-get update && apt-get install -y wget; fi
     - wget -O gitlab-cov-fuzz "${COVFUZZ_URL_PREFIX}"/"${COVFUZZ_VERSION}"/binaries/gitlab-cov-fuzz_Linux_x86_64
diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml
new file mode 100644
index 00000000000..6888e955467
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml
@@ -0,0 +1,27 @@
+stages:
+  - build
+  - test
+  - deploy
+  - dast
+
+variables:
+  SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  DAST_API_VERSION: "1"
+  DAST_API_IMAGE: $SECURE_ANALYZERS_PREFIX/api-fuzzing:$DAST_API_VERSION
+
+dast:
+  stage: dast
+  image: $DAST_API_IMAGE
+  variables:
+    GIT_STRATEGY: none
+  allow_failure: true
+  script:
+    - /peach/analyzer-dast-api
+  artifacts:
+    when: always
+    paths:
+      - gl-assets
+      - gl-dast-api-report.json
+      - gl-*.log
+    reports:
+      dast: gl-dast-api-report.json
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index e554742735c..12c987a8d37 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -5,9 +5,11 @@
 
 include:
   - template: Terraform/Base.latest.gitlab-ci.yml  # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+  - template: Jobs/SAST-IaC.latest.gitlab-ci.yml
 
 stages:
   - validate
+  - test
   - build
   - deploy
 
diff --git a/lib/gitlab/ci/trace/remote_checksum.rb b/lib/gitlab/ci/trace/remote_checksum.rb
index d57f3888ec0..7f43d91e6d7 100644
--- a/lib/gitlab/ci/trace/remote_checksum.rb
+++ b/lib/gitlab/ci/trace/remote_checksum.rb
@@ -26,7 +26,6 @@ module Gitlab
         delegate :aws?, :google?, to: :object_store_config, prefix: :provider
 
         def fetch_md5_checksum
-          return unless Feature.enabled?(:ci_archived_build_trace_checksum, trace_artifact.project, default_enabled: :yaml)
           return unless object_store_config.enabled?
           return if trace_artifact.local_store?
 
diff --git a/lib/gitlab/ci/trace/stream.rb b/lib/gitlab/ci/trace/stream.rb
index 2d31049a0c9..dd435ba05b7 100644
--- a/lib/gitlab/ci/trace/stream.rb
+++ b/lib/gitlab/ci/trace/stream.rb
@@ -11,10 +11,6 @@ module Gitlab
 
         delegate :close, :tell, :seek, :size, :url, :truncate, to: :stream, allow_nil: true
 
-        delegate :valid?, to: :stream, allow_nil: true
-
-        alias_method :present?, :valid?
-
         def initialize(metrics = Trace::Metrics.new)
           @stream = yield
           @stream&.binmode
@@ -24,6 +20,7 @@ module Gitlab
         def valid?
           self.stream.present?
         end
+        alias_method :present?, :valid?
 
         def file?
           self.path.present?
diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb
index 3e2c2c7fc1a..4c777527ebc 100644
--- a/lib/gitlab/ci/variables/builder.rb
+++ b/lib/gitlab/ci/variables/builder.rb
@@ -13,12 +13,76 @@ module Gitlab
         def scoped_variables(job, environment:, dependencies:)
           Gitlab::Ci::Variables::Collection.new.tap do |variables|
             variables.concat(predefined_variables(job))
+
+            next variables unless pipeline.use_variables_builder_definitions?
+
+            variables.concat(project.predefined_variables)
+            variables.concat(pipeline.predefined_variables)
+            variables.concat(job.runner.predefined_variables) if job.runnable? && job.runner
+            variables.concat(kubernetes_variables(job))
+            variables.concat(deployment_variables(environment: environment, job: job))
+            variables.concat(job.yaml_variables)
+            variables.concat(user_variables(job.user))
+            variables.concat(job.dependency_variables) if dependencies
+            variables.concat(secret_instance_variables(ref: job.git_ref))
+            variables.concat(secret_group_variables(environment: environment, ref: job.git_ref))
+            variables.concat(secret_project_variables(environment: environment, ref: job.git_ref))
+            variables.concat(job.trigger_request.user_variables) if job.trigger_request
+            variables.concat(pipeline.variables)
+            variables.concat(pipeline.pipeline_schedule.job_variables) if pipeline.pipeline_schedule
+          end
+        end
+
+        def kubernetes_variables(job)
+          ::Gitlab::Ci::Variables::Collection.new.tap do |collection|
+            # Should get merged with the cluster kubeconfig in deployment_variables, see
+            # https://gitlab.com/gitlab-org/gitlab/-/issues/335089
+            template = ::Ci::GenerateKubeconfigService.new(job).execute
+
+            if template.valid?
+              collection.append(key: 'KUBECONFIG', value: template.to_yaml, public: false, file: true)
+            end
           end
         end
 
+        def deployment_variables(environment:, job:)
+          return [] unless environment
+
+          project.deployment_variables(
+            environment: environment,
+            kubernetes_namespace: job.expanded_kubernetes_namespace
+          )
+        end
+
+        def user_variables(user)
+          Gitlab::Ci::Variables::Collection.new.tap do |variables|
+            break variables if user.blank?
+
+            variables.append(key: 'GITLAB_USER_ID', value: user.id.to_s)
+            variables.append(key: 'GITLAB_USER_EMAIL', value: user.email)
+            variables.append(key: 'GITLAB_USER_LOGIN', value: user.username)
+            variables.append(key: 'GITLAB_USER_NAME', value: user.name)
+          end
+        end
+
+        def secret_instance_variables(ref:)
+          project.ci_instance_variables_for(ref: ref)
+        end
+
+        def secret_group_variables(environment:, ref:)
+          return [] unless project.group
+
+          project.group.ci_variables_for(ref, project, environment: environment)
+        end
+
+        def secret_project_variables(environment:, ref:)
+          project.ci_variables_for(ref: ref, environment: environment)
+        end
+
         private
 
         attr_reader :pipeline
+        delegate :project, to: :pipeline
 
         def predefined_variables(job)
           Gitlab::Ci::Variables::Collection.new.tap do |variables|
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 296b0cfded2..553508c8638 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -86,11 +86,19 @@ module Gitlab
       def validate_job_needs!(name, job)
         return unless needs = job.dig(:needs, :job)
 
+        validate_duplicate_needs!(name, needs)
+
         needs.each do |need|
           validate_job_dependency!(name, need[:name], 'need')
         end
       end
 
+      def validate_duplicate_needs!(name, needs)
+        unless needs.uniq == needs
+          error!("#{name} has duplicate entries in the needs section.")
+        end
+      end
+
       def validate_job_dependency!(name, dependency, dependency_type = 'dependency')
         unless @jobs[dependency.to_sym]
           error!("#{name} job: undefined #{dependency_type}: #{dependency}")