Add latest changes from gitlab-org/gitlab@14-6-stable-eev14.6.0-rc42

47 files changed, 648 insertions, 137 deletions

diff --git a/lib/gitlab/ci/badge/coverage/report.rb b/lib/gitlab/ci/badge/coverage/report.rb
index 78b51dbdaf0..4d1193176ad 100644
--- a/lib/gitlab/ci/badge/coverage/report.rb
+++ b/lib/gitlab/ci/badge/coverage/report.rb
@@ -30,7 +30,7 @@ module Gitlab::Ci
           @coverage ||= raw_coverage
           return unless @coverage
 
-          @coverage.to_f.round(2)
+          @coverage.round(2)
         end
 
         def metadata
diff --git a/lib/gitlab/ci/badge/metadata.rb b/lib/gitlab/ci/badge/metadata.rb
index eec9fedfaa9..244e3aff851 100644
--- a/lib/gitlab/ci/badge/metadata.rb
+++ b/lib/gitlab/ci/badge/metadata.rb
@@ -8,14 +8,13 @@ module Gitlab::Ci
     class Metadata
       include Gitlab::Routing
       include ActionView::Helpers::AssetTagHelper
-      include ActionView::Helpers::UrlHelper
 
       def initialize(badge)
         @badge = badge
       end
 
       def to_html
-        link_to(image_tag(image_url, alt: title), link_url)
+        ApplicationController.helpers.link_to(image_tag(image_url, alt: title), link_url)
       end
 
       def to_markdown
diff --git a/lib/gitlab/ci/build/context/base.rb b/lib/gitlab/ci/build/context/base.rb
index c7ea7c78e2f..81f96e822f4 100644
--- a/lib/gitlab/ci/build/context/base.rb
+++ b/lib/gitlab/ci/build/context/base.rb
@@ -17,6 +17,12 @@ module Gitlab
             raise NotImplementedError
           end
 
+          def variables_hash
+            strong_memoize(:variables_hash) do
+              variables.to_hash
+            end
+          end
+
           def project
             pipeline.project
           end
diff --git a/lib/gitlab/ci/build/policy/variables.rb b/lib/gitlab/ci/build/policy/variables.rb
index 7b1ce6330f0..810523052ae 100644
--- a/lib/gitlab/ci/build/policy/variables.rb
+++ b/lib/gitlab/ci/build/policy/variables.rb
@@ -10,7 +10,7 @@ module Gitlab
           end
 
           def satisfied_by?(pipeline, context)
-            variables = context.variables
+            variables = context.variables_hash
 
             statements = @expressions.map do |statement|
               ::Gitlab::Ci::Pipeline::Expression::Statement
diff --git a/lib/gitlab/ci/build/rules/rule/clause/changes.rb b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
index 9c2f6eea1dd..82a59fdb4e1 100644
--- a/lib/gitlab/ci/build/rules/rule/clause/changes.rb
+++ b/lib/gitlab/ci/build/rules/rule/clause/changes.rb
@@ -23,7 +23,7 @@ module Gitlab
           return @globs unless context
 
           @globs.map do |glob|
-            ExpandVariables.expand_existing(glob, context.variables)
+            ExpandVariables.expand_existing(glob, -> { context.variables_hash })
           end
         end
       end
diff --git a/lib/gitlab/ci/build/rules/rule/clause/if.rb b/lib/gitlab/ci/build/rules/rule/clause/if.rb
index 6143a736ca6..499a265a1e2 100644
--- a/lib/gitlab/ci/build/rules/rule/clause/if.rb
+++ b/lib/gitlab/ci/build/rules/rule/clause/if.rb
@@ -10,7 +10,7 @@ module Gitlab
 
         def satisfied_by?(pipeline, context)
           ::Gitlab::Ci::Pipeline::Expression::Statement.new(
-            @expression, context.variables).truthful?
+            @expression, context.variables_hash).truthful?
         end
       end
     end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 6f149385969..42b487fdf81 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -17,21 +17,27 @@ module Gitlab
         Config::Yaml::Tags::TagError
       ].freeze
 
-      attr_reader :root, :context, :source_ref_path, :source
+      attr_reader :root, :context, :source_ref_path, :source, :logger
 
-      def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil)
+      def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil)
+        @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project)
         @source_ref_path = pipeline&.source_ref_path
 
-        @context = build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
+        @context = self.logger.instrument(:config_build_context) do
+          build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline)
+        end
+
         @context.set_deadline(TIMEOUT_SECONDS)
 
         @source = source
 
-        @config = expand_config(config)
-
-        @root = Entry::Root.new(@config)
-        @root.compose!
+        @config = self.logger.instrument(:config_expand) do
+          expand_config(config)
+        end
 
+        @root = self.logger.instrument(:config_compose) do
+          Entry::Root.new(@config).tap(&:compose!)
+        end
       rescue *rescue_errors => e
         raise Config::ConfigError, e.message
       end
@@ -94,11 +100,25 @@ module Gitlab
       end
 
       def build_config(config)
-        initial_config = Config::Yaml.load!(config)
-        initial_config = Config::External::Processor.new(initial_config, @context).perform
-        initial_config = Config::Extendable.new(initial_config).to_hash
-        initial_config = Config::Yaml::Tags::Resolver.new(initial_config).to_hash
-        Config::EdgeStagesInjector.new(initial_config).to_hash
+        initial_config = logger.instrument(:config_yaml_load) do
+          Config::Yaml.load!(config)
+        end
+
+        initial_config = logger.instrument(:config_external_process) do
+          Config::External::Processor.new(initial_config, @context).perform
+        end
+
+        initial_config = logger.instrument(:config_yaml_extend) do
+          Config::Extendable.new(initial_config).to_hash
+        end
+
+        initial_config = logger.instrument(:config_tags_resolve) do
+          Config::Yaml::Tags::Resolver.new(initial_config).to_hash
+        end
+
+        logger.instrument(:config_stages_inject) do
+          Config::EdgeStagesInjector.new(initial_config).to_hash
+        end
       end
 
       def find_sha(project)
@@ -115,10 +135,20 @@ module Gitlab
           sha: sha || find_sha(project),
           user: user,
           parent_pipeline: parent_pipeline,
-          variables: build_variables(project: project, pipeline: pipeline))
+          variables: build_variables(project: project, pipeline: pipeline),
+          logger: logger)
       end
 
       def build_variables(project:, pipeline:)
+        logger.instrument(:config_build_variables) do
+          build_variables_without_instrumentation(
+            project: project,
+            pipeline: pipeline
+          )
+        end
+      end
+
+      def build_variables_without_instrumentation(project:, pipeline:)
         Gitlab::Ci::Variables::Collection.new.tap do |variables|
           break variables unless project
 
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 520b1ce6119..43475742214 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -26,7 +26,7 @@ module Gitlab
               validates :name, length: { maximum: 255 }, if: -> { ::Feature.enabled?(:ci_validate_job_length, default_enabled: :yaml) }
 
               validates :config, disallowed_keys: {
-                  in: %i[only except when start_in],
+                  in: %i[only except start_in],
                   message: 'key may not be used with `rules`'
                 },
                 if: :has_rules?
diff --git a/lib/gitlab/ci/config/entry/tags.rb b/lib/gitlab/ci/config/entry/tags.rb
index ca3b48372e2..6044cfddbdc 100644
--- a/lib/gitlab/ci/config/entry/tags.rb
+++ b/lib/gitlab/ci/config/entry/tags.rb
@@ -16,8 +16,6 @@ module Gitlab
             validates :config, array_of_strings: true
 
             validate do
-              next unless ::Feature.enabled?(:ci_build_tags_limit, default_enabled: :yaml)
-
               if config.is_a?(Array) && config.size >= TAGS_LIMIT
                 errors.add(:config, _("must be less than the limit of %{tag_limit} tags") % { tag_limit: TAGS_LIMIT })
               end
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index 51624dc30ea..308414af47d 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -9,17 +9,22 @@ module Gitlab
 
           TimeoutError = Class.new(StandardError)
 
+          include ::Gitlab::Utils::StrongMemoize
+
           attr_reader :project, :sha, :user, :parent_pipeline, :variables
-          attr_reader :expandset, :execution_deadline
+          attr_reader :expandset, :execution_deadline, :logger
+
+          delegate :instrument, to: :logger
 
-          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: [])
+          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, logger: nil)
             @project = project
             @sha = sha
             @user = user
             @parent_pipeline = parent_pipeline
-            @variables = variables
+            @variables = variables || Ci::Variables::Collection.new
             @expandset = Set.new
             @execution_deadline = 0
+            @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project)
 
             yield self if block_given?
           end
@@ -36,10 +41,17 @@ module Gitlab
             end
           end
 
+          def variables_hash
+            strong_memoize(:variables_hash) do
+              variables.to_hash
+            end
+          end
+
           def mutate(attrs = {})
             self.class.new(**attrs) do |ctx|
               ctx.expandset = expandset
               ctx.execution_deadline = execution_deadline
+              ctx.logger = logger
             end
           end
 
@@ -60,7 +72,7 @@ module Gitlab
 
           protected
 
-          attr_writer :expandset, :execution_deadline
+          attr_writer :expandset, :execution_deadline, :logger
 
           private
 
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 95f1a842c50..a5bf066c81f 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -30,6 +30,18 @@ module Gitlab
           def process
             return [] if locations.empty?
 
+            logger.instrument(:config_mapper_process) do
+              process_without_instrumentation
+            end
+          end
+
+          private
+
+          attr_reader :locations, :context
+
+          delegate :expandset, :logger, to: :context
+
+          def process_without_instrumentation
             locations
               .compact
               .map(&method(:normalize_location))
@@ -41,14 +53,14 @@ module Gitlab
               .map(&method(:select_first_matching))
           end
 
-          private
-
-          attr_reader :locations, :context
-
-          delegate :expandset, to: :context
+          def normalize_location(location)
+            logger.instrument(:config_mapper_normalize) do
+              normalize_location_without_instrumentation(location)
+            end
+          end
 
           # convert location if String to canonical form
-          def normalize_location(location)
+          def normalize_location_without_instrumentation(location)
             if location.is_a?(String)
               expanded_location = expand_variables(location)
               normalize_location_string(expanded_location)
@@ -58,6 +70,12 @@ module Gitlab
           end
 
           def verify_rules(location)
+            logger.instrument(:config_mapper_rules) do
+              verify_rules_without_instrumentation(location)
+            end
+          end
+
+          def verify_rules_without_instrumentation(location)
             return unless Rules.new(location[:rules]).evaluate(context).pass?
 
             location
@@ -72,6 +90,12 @@ module Gitlab
           end
 
           def expand_wildcard_paths(location)
+            logger.instrument(:config_mapper_wildcards) do
+              expand_wildcard_paths_without_instrumentation(location)
+            end
+          end
+
+          def expand_wildcard_paths_without_instrumentation(location)
             # We only support local files for wildcard paths
             return location unless location[:local] && location[:local].include?('*')
 
@@ -89,6 +113,12 @@ module Gitlab
           end
 
           def verify_duplicates!(location)
+            logger.instrument(:config_mapper_verify) do
+              verify_duplicates_without_instrumentation!(location)
+            end
+          end
+
+          def verify_duplicates_without_instrumentation!(location)
             if expandset.count >= MAX_INCLUDES
               raise TooManyIncludesError, "Maximum of #{MAX_INCLUDES} nested includes are allowed!"
             end
@@ -106,6 +136,12 @@ module Gitlab
           end
 
           def select_first_matching(location)
+            logger.instrument(:config_mapper_select) do
+              select_first_matching_without_instrumentation(location)
+            end
+          end
+
+          def select_first_matching_without_instrumentation(location)
             matching = FILE_CLASSES.map do |file_class|
               file_class.new(location, context)
             end.select(&:matching?)
@@ -116,6 +152,12 @@ module Gitlab
           end
 
           def expand_variables(data)
+            logger.instrument(:config_mapper_variables) do
+              expand_variables_without_instrumentation(data)
+            end
+          end
+
+          def expand_variables_without_instrumentation(data)
             if data.is_a?(String)
               expand(data)
             else
@@ -137,7 +179,7 @@ module Gitlab
           end
 
           def expand(data)
-            ExpandVariables.expand(data, context.variables)
+            ExpandVariables.expand(data, -> { context.variables_hash })
           end
         end
       end
diff --git a/lib/gitlab/ci/config/external/processor.rb b/lib/gitlab/ci/config/external/processor.rb
index de69a1b1e8f..6a4aee26d80 100644
--- a/lib/gitlab/ci/config/external/processor.rb
+++ b/lib/gitlab/ci/config/external/processor.rb
@@ -7,10 +7,13 @@ module Gitlab
         class Processor
           IncludeError = Class.new(StandardError)
 
+          attr_reader :context, :logger
+
           def initialize(values, context)
             @values = values
             @external_files = External::Mapper.new(values, context).process
             @content = {}
+            @logger = context.logger
           rescue External::Mapper::Error,
                  OpenSSL::SSL::SSLError => e
             raise IncludeError, e.message
@@ -29,13 +32,17 @@ module Gitlab
 
           def validate_external_files!
             @external_files.each do |file|
-              raise IncludeError, file.error_message unless file.valid?
+              logger.instrument(:config_external_verify) do
+                raise IncludeError, file.error_message unless file.valid?
+              end
             end
           end
 
           def merge_external_files!
             @external_files.each do |file|
-              @content.deep_merge!(file.to_hash)
+              logger.instrument(:config_external_merge) do
+                @content.deep_merge!(file.to_hash)
+              end
             end
           end
 
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
deleted file mode 100644
index 51051b0490f..00000000000
--- a/lib/gitlab/ci/features.rb
+++ /dev/null
@@ -1,30 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
-  module Ci
-    ##
-    # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place.
-    #
-    module Features
-      # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
-      # is a safe switch to disable the feature for a particular project when something went wrong,
-      # therefore it's not supposed to be enabled by default.
-      def self.disallow_to_create_merge_request_pipelines_in_target_project?(target_project)
-        ::Feature.enabled?(:ci_disallow_to_create_merge_request_pipelines_in_target_project, target_project)
-      end
-
-      def self.accept_trace?(project)
-        ::Feature.enabled?(:ci_enable_live_trace, project) &&
-          ::Feature.enabled?(:ci_accept_trace, project, type: :ops, default_enabled: true)
-      end
-
-      def self.log_invalid_trace_chunks?(project)
-        ::Feature.enabled?(:ci_trace_log_invalid_chunks, project, type: :ops, default_enabled: false)
-      end
-
-      def self.gldropdown_tags_enabled?
-        ::Feature.enabled?(:gldropdown_tags, default_enabled: :yaml)
-      end
-    end
-  end
-end
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index 73cfa02ce4b..651ed23eb25 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -34,7 +34,7 @@ module Gitlab
               end
 
               def file_name
-                "#{report_type.to_s.dasherize}-report-format.json"
+                report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json"
               end
             end
 
diff --git a/lib/gitlab/ci/parsers/terraform/tfplan.rb b/lib/gitlab/ci/parsers/terraform/tfplan.rb
index f9afa58f915..041d73cd914 100644
--- a/lib/gitlab/ci/parsers/terraform/tfplan.rb
+++ b/lib/gitlab/ci/parsers/terraform/tfplan.rb
@@ -34,7 +34,7 @@ module Gitlab
           def job_details(job)
             {
               'job_id' => job.id.to_s,
-              'job_name' => job.options.dig(:artifacts, :name).to_s,
+              'job_name' => job.name,
               'job_path' => Gitlab::Routing.url_helpers.project_job_path(job.project, job)
             }
           end
diff --git a/lib/gitlab/ci/pipeline/chain/base.rb b/lib/gitlab/ci/pipeline/chain/base.rb
index 9b494f3a7ec..28567437719 100644
--- a/lib/gitlab/ci/pipeline/chain/base.rb
+++ b/lib/gitlab/ci/pipeline/chain/base.rb
@@ -7,7 +7,7 @@ module Gitlab
         class Base
           attr_reader :pipeline, :command, :config
 
-          delegate :project, :current_user, :parent_pipeline, to: :command
+          delegate :project, :current_user, :parent_pipeline, :logger, to: :command
 
           def initialize(pipeline, command)
             @pipeline = pipeline
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index 6feb693221b..bbdc6b65b96 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -21,6 +21,10 @@ module Gitlab
               merge_request: @command.merge_request,
               external_pull_request: @command.external_pull_request,
               locked: @command.project.default_pipeline_lock)
+
+            # Initialize the feature flag at the beginning of the pipeline creation process
+            # so that the flag references in the latter chains return the same value.
+            @pipeline.create_deployment_in_separate_transaction?
           end
 
           def break?
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index beb8801096b..c466b8b36d0 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -11,7 +11,7 @@ module Gitlab
           :trigger_request, :schedule, :merge_request, :external_pull_request,
           :ignore_skip_ci, :save_incompleted,
           :seeds_block, :variables_attributes, :push_options,
-          :chat_data, :allow_mirror_update, :bridge, :content, :dry_run,
+          :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger,
           # These attributes are set by Chains during processing:
           :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed
         ) do
@@ -88,7 +88,14 @@ module Gitlab
             @metrics ||= ::Gitlab::Ci::Pipeline::Metrics
           end
 
+          def logger
+            self[:logger] ||= ::Gitlab::Ci::Pipeline::Logger.new(project: project)
+          end
+
           def observe_step_duration(step_class, duration)
+            step = step_class.name.underscore.parameterize(separator: '_')
+            logger.observe("pipeline_step_#{step}_duration_s", duration)
+
             if Feature.enabled?(:ci_pipeline_creation_step_duration_tracking, type: :ops, default_enabled: :yaml)
               metrics.pipeline_creation_step_duration_histogram
                 .observe({ step: step_class.name }, duration.seconds)
@@ -96,11 +103,15 @@ module Gitlab
           end
 
           def observe_creation_duration(duration)
+            logger.observe(:pipeline_creation_duration_s, duration)
+
             metrics.pipeline_creation_duration_histogram
               .observe({}, duration.seconds)
           end
 
           def observe_pipeline_size(pipeline)
+            logger.observe(:pipeline_size_count, pipeline.total_size)
+
             metrics.pipeline_size_histogram
               .observe({ source: pipeline.source.to_s }, pipeline.total_size)
           end
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index f3c937ddd28..64d1b001e3c 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -11,16 +11,21 @@ module Gitlab
             def perform!
               raise ArgumentError, 'missing config content' unless @command.config_content
 
-              result = ::Gitlab::Ci::YamlProcessor.new(
-                @command.config_content, {
-                  project: project,
-                  pipeline: @pipeline,
-                  sha: @pipeline.sha,
-                  source: @pipeline.source,
-                  user: current_user,
-                  parent_pipeline: parent_pipeline
-                }
-              ).execute
+              result = logger.instrument(:pipeline_config_process) do
+                processor = ::Gitlab::Ci::YamlProcessor.new(
+                  @command.config_content, {
+                    project: project,
+                    pipeline: @pipeline,
+                    sha: @pipeline.sha,
+                    source: @pipeline.source,
+                    user: current_user,
+                    parent_pipeline: parent_pipeline,
+                    logger: logger
+                  }
+                )
+
+                processor.execute
+              end
 
               add_warnings_to_pipeline(result.warnings)
 
diff --git a/lib/gitlab/ci/pipeline/chain/create.rb b/lib/gitlab/ci/pipeline/chain/create.rb
index 81ef3bb074d..15b0ff3c04d 100644
--- a/lib/gitlab/ci/pipeline/chain/create.rb
+++ b/lib/gitlab/ci/pipeline/chain/create.rb
@@ -6,10 +6,18 @@ module Gitlab
       module Chain
         class Create < Chain::Base
           include Chain::Helpers
+          include Gitlab::Utils::StrongMemoize
 
           def perform!
-            BulkInsertableAssociations.with_bulk_insert do
-              pipeline.save!
+            logger.instrument(:pipeline_save) do
+              BulkInsertableAssociations.with_bulk_insert do
+                tags = extract_tag_list_by_status
+
+                pipeline.transaction do
+                  pipeline.save!
+                  CommitStatus.bulk_insert_tags!(statuses, tags) if bulk_insert_tags?
+                end
+              end
             end
           rescue ActiveRecord::RecordInvalid => e
             error("Failed to persist the pipeline: #{e}")
@@ -18,6 +26,37 @@ module Gitlab
           def break?
             !pipeline.persisted?
           end
+
+          private
+
+          def statuses
+            strong_memoize(:statuses) do
+              pipeline.stages.flat_map(&:statuses)
+            end
+          end
+
+          # We call `job.tag_list=` to assign tags to the jobs from the
+          # Chain::Seed step which uses the `@tag_list` instance variable to
+          # store them on the record. We remove them here because we want to
+          # bulk insert them, otherwise they would be inserted and assigned one
+          # by one with callbacks. We must use `remove_instance_variable`
+          # because having the instance variable defined would still run the callbacks
+          def extract_tag_list_by_status
+            return {} unless bulk_insert_tags?
+
+            statuses.each.with_object({}) do |job, acc|
+              tag_list = job.clear_memoization(:tag_list)
+              next unless tag_list
+
+              acc[job.name] = tag_list
+            end
+          end
+
+          def bulk_insert_tags?
+            strong_memoize(:bulk_insert_tags) do
+              ::Feature.enabled?(:ci_bulk_insert_tags, project, default_enabled: :yaml)
+            end
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/create_deployments.rb b/lib/gitlab/ci/pipeline/chain/create_deployments.rb
new file mode 100644
index 00000000000..b92aa89d62d
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/create_deployments.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class CreateDeployments < Chain::Base
+          DeploymentCreationError = Class.new(StandardError)
+
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            create_deployments!
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def create_deployments!
+            pipeline.stages.map(&:statuses).flatten.map(&method(:create_deployment))
+          end
+
+          def create_deployment(build)
+            return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present?
+
+            deployment = ::Gitlab::Ci::Pipeline::Seed::Deployment
+              .new(build, build.persisted_environment).to_resource
+
+            return unless deployment
+
+            deployment.deployable = build
+            deployment.save!
+          rescue ActiveRecord::RecordInvalid => e
+            Gitlab::ErrorTracking.track_and_raise_for_dev_exception(
+              DeploymentCreationError.new(e.message), build_id: build.id)
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb
new file mode 100644
index 00000000000..424e1d87fb4
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class EnsureEnvironments < Chain::Base
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_environment))
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def ensure_environment(build)
+            return unless build.instance_of?(::Ci::Build) && build.has_environment?
+
+            environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource
+
+            if environment.persisted?
+              build.persisted_environment = environment
+              build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name })
+            else
+              build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb
new file mode 100644
index 00000000000..f4e5e6e467a
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/ensure_resource_groups.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class EnsureResourceGroups < Chain::Base
+          def perform!
+            return unless pipeline.create_deployment_in_separate_transaction?
+
+            pipeline.stages.map(&:statuses).flatten.each(&method(:ensure_resource_group))
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def ensure_resource_group(processable)
+            return unless processable.is_a?(::Ci::Processable)
+
+            key = processable.options.delete(:resource_group_key)
+
+            resource_group = ::Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup
+              .new(processable, key).to_resource
+
+            processable.resource_group = resource_group
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index ef7447fa83d..356eeb76908 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -13,8 +13,10 @@ module Gitlab
             raise ArgumentError, 'missing workflow rules result' unless @command.workflow_rules_result
 
             # Allocate next IID. This operation must be outside of transactions of pipeline creations.
-            pipeline.ensure_project_iid!
-            pipeline.ensure_ci_ref!
+            logger.instrument(:pipeline_allocate_seed_attributes) do
+              pipeline.ensure_project_iid!
+              pipeline.ensure_ci_ref!
+            end
 
             # Protect the pipeline. This is assigned in Populate instead of
             # Build to prevent erroring out on ambiguous refs.
@@ -23,8 +25,12 @@ module Gitlab
             ##
             # Gather all runtime build/stage errors
             #
-            if pipeline_seed.errors
-              return error(pipeline_seed.errors.join("\n"), config_error: true)
+            seed_errors = logger.instrument(:pipeline_seed_evaluation) do
+              pipeline_seed.errors
+            end
+
+            if seed_errors
+              return error(seed_errors.join("\n"), config_error: true)
             end
 
             @command.pipeline_seed = pipeline_seed
@@ -38,8 +44,11 @@ module Gitlab
 
           def pipeline_seed
             strong_memoize(:pipeline_seed) do
-              stages_attributes = @command.yaml_processor_result.stages_attributes
-              Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes)
+              logger.instrument(:pipeline_seed_initialization) do
+                stages_attributes = @command.yaml_processor_result.stages_attributes
+
+                Gitlab::Ci::Pipeline::Seed::Pipeline.new(context, stages_attributes)
+              end
             end
           end
 
@@ -48,9 +57,11 @@ module Gitlab
           end
 
           def root_variables
-            ::Gitlab::Ci::Variables::Helpers.merge_variables(
-              @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables
-            )
+            logger.instrument(:pipeline_seed_merge_variables) do
+              ::Gitlab::Ci::Variables::Helpers.merge_variables(
+                @command.yaml_processor_result.root_variables, @command.workflow_rules_result.variables
+              )
+            end
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index 845eb6c7a42..de147914850 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -9,30 +9,36 @@ module Gitlab
             @pipeline = pipeline
             @command = command
             @sequence = sequence
-            @start = Time.now
+            @start = current_monotonic_time
           end
 
           def build!
             @sequence.each do |step_class|
-              step_start = ::Gitlab::Metrics::System.monotonic_time
+              step_start = current_monotonic_time
               step = step_class.new(@pipeline, @command)
 
               step.perform!
 
               @command.observe_step_duration(
                 step_class,
-                ::Gitlab::Metrics::System.monotonic_time - step_start
+                current_monotonic_time - step_start
               )
 
               break if step.break?
             end
 
-            @command.observe_creation_duration(Time.now - @start)
+            @command.observe_creation_duration(current_monotonic_time - @start)
             @command.observe_pipeline_size(@pipeline)
             @command.observe_jobs_count_in_alive_pipelines
 
             @pipeline
           end
+
+          private
+
+          def current_monotonic_time
+            ::Gitlab::Metrics::System.monotonic_time
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb
index 28ba1cd4d47..85bd5f0a7c1 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/external.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb
@@ -113,7 +113,7 @@ module Gitlab
                 name: build[:name],
                 stage: build[:stage],
                 image: build.dig(:options, :image, :name),
-                services: build.dig(:options, :services)&.map { |service| service[:name] },
+                services: service_names(build),
                 script: [
                   build.dig(:options, :before_script),
                   build.dig(:options, :script),
@@ -122,6 +122,14 @@ module Gitlab
               }
             end
 
+            def service_names(build)
+              services = build.dig(:options, :services)
+
+              return unless services
+
+              services.compact.map { |service| service[:name] }
+            end
+
             def stages_attributes
               command.yaml_processor_result.stages_attributes
             end
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
index 11d2010909f..6da88fd287e 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/variable.rb
@@ -9,7 +9,11 @@ module Gitlab
             PATTERN = /\$(?<name>\w+)/.freeze
 
             def evaluate(variables = {})
-              variables.with_indifferent_access.fetch(@value, nil)
+              unless variables.is_a?(ActiveSupport::HashWithIndifferentAccess)
+                variables = variables.with_indifferent_access
+              end
+
+              variables.fetch(@value, nil)
             end
 
             def inspect
diff --git a/lib/gitlab/ci/pipeline/expression/statement.rb b/lib/gitlab/ci/pipeline/expression/statement.rb
index 5f3310dd668..4b13cae792e 100644
--- a/lib/gitlab/ci/pipeline/expression/statement.rb
+++ b/lib/gitlab/ci/pipeline/expression/statement.rb
@@ -9,7 +9,7 @@ module Gitlab
 
           def initialize(statement, variables = nil)
             @lexer = Expression::Lexer.new(statement)
-            @variables = variables&.to_hash
+            @variables = variables || {}
           end
 
           def parse_tree
@@ -19,7 +19,7 @@ module Gitlab
           end
 
           def evaluate
-            parse_tree.evaluate(@variables.to_h)
+            parse_tree.evaluate(@variables)
           end
 
           def truthful?
diff --git a/lib/gitlab/ci/pipeline/logger.rb b/lib/gitlab/ci/pipeline/logger.rb
new file mode 100644
index 00000000000..97f7dddd09a
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/logger.rb
@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      class Logger
+        include ::Gitlab::Utils::StrongMemoize
+
+        def self.current_monotonic_time
+          ::Gitlab::Metrics::System.monotonic_time
+        end
+
+        def initialize(project:, destination: Gitlab::AppJsonLogger)
+          @started_at = current_monotonic_time
+          @project = project
+          @destination = destination
+          @log_conditions = []
+
+          yield(self) if block_given?
+        end
+
+        def log_when(&block)
+          log_conditions.push(block)
+        end
+
+        def instrument(operation)
+          return yield unless enabled?
+
+          raise ArgumentError, 'block not given' unless block_given?
+
+          op_started_at = current_monotonic_time
+
+          result = yield
+
+          observe("#{operation}_duration_s", current_monotonic_time - op_started_at)
+
+          result
+        end
+
+        def observe(operation, value)
+          return unless enabled?
+
+          observations[operation.to_s].push(value)
+        end
+
+        def commit(pipeline:, caller:)
+          return unless log?
+
+          attributes = {
+            class: self.class.name.to_s,
+            pipeline_creation_caller: caller,
+            project_id: project.id,
+            pipeline_id: pipeline.id,
+            pipeline_persisted: pipeline.persisted?,
+            pipeline_source: pipeline.source,
+            pipeline_creation_service_duration_s: age
+          }.stringify_keys.merge(observations_hash)
+
+          destination.info(attributes)
+        end
+
+        def observations_hash
+          observations.transform_values do |values|
+            next if values.empty?
+
+            {
+              'count' => values.size,
+              'min' => values.min,
+              'max' => values.max,
+              'avg' => values.sum / values.size
+            }
+          end.compact
+        end
+
+        private
+
+        attr_reader :project, :destination, :started_at, :log_conditions
+        delegate :current_monotonic_time, to: :class
+
+        def age
+          current_monotonic_time - started_at
+        end
+
+        def log?
+          return false unless enabled?
+          return true if log_conditions.empty?
+
+          log_conditions.any? { |cond| cond.call(observations) }
+        end
+
+        def enabled?
+          strong_memoize(:enabled) do
+            ::Feature.enabled?(:ci_pipeline_creation_logger, project, type: :ops, default_enabled: :yaml)
+          end
+        end
+
+        def observations
+          @observations ||= Hash.new { |hash, key| hash[key] = [] }
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 72837b8ec22..762292f0fa3 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -7,8 +7,6 @@ module Gitlab
         class Build < Seed::Base
           include Gitlab::Utils::StrongMemoize
 
-          EnvironmentCreationFailure = Class.new(StandardError)
-
           delegate :dig, to: :@seed_attributes
 
           def initialize(context, attributes, stages_for_needs_lookup = [])
@@ -30,7 +28,7 @@ module Gitlab
             @except = Gitlab::Ci::Build::Policy
               .fabricate(attributes.delete(:except))
             @rules = Gitlab::Ci::Build::Rules
-              .new(attributes.delete(:rules), default_when: 'on_success')
+              .new(attributes.delete(:rules), default_when: attributes[:when])
             @cache = Gitlab::Ci::Build::Cache
               .new(attributes.delete(:cache), @pipeline)
 
@@ -80,7 +78,7 @@ module Gitlab
           def to_resource
             strong_memoize(:resource) do
               processable = initialize_processable
-              assign_resource_group(processable)
+              assign_resource_group(processable) unless @pipeline.create_deployment_in_separate_transaction?
               processable
             end
           end
@@ -90,7 +88,9 @@ module Gitlab
               ::Ci::Bridge.new(attributes)
             else
               ::Ci::Build.new(attributes).tap do |build|
-                build.assign_attributes(self.class.deployment_attributes_for(build))
+                unless @pipeline.create_deployment_in_separate_transaction?
+                  build.assign_attributes(self.class.deployment_attributes_for(build))
+                end
               end
             end
           end
@@ -107,20 +107,7 @@ module Gitlab
             environment = Seed::Environment.new(build).to_resource if environment.nil?
 
             unless environment.persisted?
-              if Feature.enabled?(:surface_environment_creation_failure, build.project, default_enabled: :yaml) &&
-                 Feature.disabled?(:surface_environment_creation_failure_override, build.project)
-                return { status: :failed, failure_reason: :environment_creation_failure }
-              end
-
-              # If there is a validation error on environment creation, such as
-              # the name contains invalid character, the build falls back to a
-              # non-environment job.
-              Gitlab::ErrorTracking.track_exception(
-                EnvironmentCreationFailure.new,
-                project_id: build.project_id,
-                reason: environment.errors.full_messages.to_sentence)
-
-              return { environment: nil }
+              return { status: :failed, failure_reason: :environment_creation_failure }
             end
 
             build.persisted_environment = environment
@@ -215,12 +202,14 @@ module Gitlab
           end
 
           def runner_tags
-            { tag_list: evaluate_runner_tags }.compact
+            strong_memoize(:runner_tags) do
+              { tag_list: evaluate_runner_tags }.compact
+            end
           end
 
           def evaluate_runner_tags
-            @seed_attributes[:tag_list]&.map do |tag|
-              ExpandVariables.expand_existing(tag, evaluate_context.variables)
+            @seed_attributes.delete(:tag_list)&.map do |tag|
+              ExpandVariables.expand_existing(tag, -> { evaluate_context.variables_hash })
             end
           end
 
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
index 417319cb5be..3e4a44a2e70 100644
--- a/lib/gitlab/ci/reports/security/report.rb
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -51,7 +51,7 @@ module Gitlab
 
           def replace_with!(other)
             instance_variables.each do |ivar|
-              instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend
+              instance_variable_set(ivar, other.public_send(ivar.to_s[1..])) # rubocop:disable GitlabSecurity/PublicSend
             end
           end
 
diff --git a/lib/gitlab/ci/status/bridge/common.rb b/lib/gitlab/ci/status/bridge/common.rb
index d66d4b20bba..eaa87157716 100644
--- a/lib/gitlab/ci/status/bridge/common.rb
+++ b/lib/gitlab/ci/status/bridge/common.rb
@@ -16,7 +16,11 @@ module Gitlab
           def details_path
             return unless can?(user, :read_pipeline, downstream_pipeline)
 
-            project_pipeline_path(downstream_project, downstream_pipeline)
+            if Feature.enabled?(:ci_retry_downstream_pipeline, subject.project, default_enabled: :yaml)
+              project_job_path(subject.project, subject)
+            else
+              project_pipeline_path(downstream_project, downstream_pipeline)
+            end
           end
 
           def has_action?
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index b0f12ff7517..5dd28157b1f 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -34,7 +34,8 @@ module Gitlab
             no_matching_runner: 'no matching runner available',
             trace_size_exceeded: 'log size limit exceeded',
             builds_disabled: 'project builds are disabled',
-            environment_creation_failure: 'environment creation failure'
+            environment_creation_failure: 'environment creation failure',
+            deployment_rejected: 'deployment rejected'
           }.freeze
 
           private_constant :REASONS
diff --git a/lib/gitlab/ci/tags/bulk_insert.rb b/lib/gitlab/ci/tags/bulk_insert.rb
new file mode 100644
index 00000000000..a299df7e2d9
--- /dev/null
+++ b/lib/gitlab/ci/tags/bulk_insert.rb
@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Tags
+      class BulkInsert
+        TAGGINGS_BATCH_SIZE = 1000
+        TAGS_BATCH_SIZE = 500
+
+        def initialize(statuses, tag_list_by_status)
+          @statuses = statuses
+          @tag_list_by_status = tag_list_by_status
+        end
+
+        def insert!
+          return false if tag_list_by_status.empty?
+
+          persist_build_tags!
+        end
+
+        private
+
+        attr_reader :statuses, :tag_list_by_status
+
+        def persist_build_tags!
+          all_tags = tag_list_by_status.values.flatten.uniq.reject(&:blank?)
+          tag_records_by_name = create_tags(all_tags).index_by(&:name)
+          taggings = build_taggings_attributes(tag_records_by_name)
+
+          return false if taggings.empty?
+
+          taggings.each_slice(TAGGINGS_BATCH_SIZE) do |taggings_slice|
+            ActsAsTaggableOn::Tagging.insert_all!(taggings)
+          end
+
+          true
+        end
+
+        # rubocop: disable CodeReuse/ActiveRecord
+        def create_tags(tags)
+          existing_tag_records = ActsAsTaggableOn::Tag.where(name: tags).to_a
+          missing_tags = detect_missing_tags(tags, existing_tag_records)
+          return existing_tag_records if missing_tags.empty?
+
+          missing_tags
+            .map { |tag| { name: tag } }
+            .each_slice(TAGS_BATCH_SIZE) do |tags_attributes|
+              ActsAsTaggableOn::Tag.insert_all!(tags_attributes)
+            end
+
+          ActsAsTaggableOn::Tag.where(name: tags).to_a
+        end
+        # rubocop: enable CodeReuse/ActiveRecord
+
+        def build_taggings_attributes(tag_records_by_name)
+          taggings = statuses.flat_map do |status|
+            tag_list = tag_list_by_status[status.name]
+            next unless tag_list
+
+            tags = tag_records_by_name.values_at(*tag_list)
+            taggings_for(tags, status)
+          end
+
+          taggings.compact!
+          taggings
+        end
+
+        def taggings_for(tags, status)
+          tags.map do |tag|
+            {
+              tag_id: tag.id,
+              taggable_type: CommitStatus.name,
+              taggable_id: status.id,
+              created_at: Time.current,
+              context: 'tags'
+            }
+          end
+        end
+
+        def detect_missing_tags(tags, tag_records)
+          if tags.size != tag_records.size
+            tags - tag_records.map(&:name)
+          else
+            []
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
index 89fd59d98f4..fddcc1492a8 100644
--- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml
@@ -53,6 +53,9 @@ variables:
   # KUBE_INGRESS_BASE_DOMAIN is the application deployment domain and should be set as a variable at the group or project level.
   # KUBE_INGRESS_BASE_DOMAIN: domain.example.com
 
+  # Allows Container-Scanning to correctly correlate image names when using Jobs/Build.gitlab-ci.yml
+  CS_DEFAULT_BRANCH_IMAGE: $CI_REGISTRY_IMAGE/$CI_DEFAULT_BRANCH:$CI_COMMIT_SHA
+
   POSTGRES_USER: user
   POSTGRES_PASSWORD: testing-password
   POSTGRES_ENABLED: "true"
diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
index b763705857e..fa7f6ffa2b7 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml
@@ -24,7 +24,7 @@ kics-iac-sast:
   image:
     name: "$SAST_ANALYZER_IMAGE"
   variables:
-    SAST_ANALYZER_IMAGE_TAG: 0
+    SAST_ANALYZER_IMAGE_TAG: 1
     SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG"
   rules:
     - if: $SAST_DISABLED
diff --git a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml
index 17ed1d2e87f..d32444833fb 100644
--- a/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Pages/HTML.gitlab-ci.yml
@@ -9,6 +9,7 @@ pages:
   script:
     - mkdir .public
     - cp -r * .public
+    - rm -rf public
     - mv .public public
   artifacts:
     paths:
diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
index aec41c137a4..4917abf6ae9 100644
--- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml
@@ -23,7 +23,7 @@ cache:
     - venv/
 
 before_script:
-  - python -V  # Print out python version for debugging
+  - python --version  # For debugging
   - pip install virtualenv
   - virtualenv venv
   - source venv/bin/activate
diff --git a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
index ff8f9601189..de54d64dc42 100644
--- a/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Scala.gitlab-ci.yml
@@ -13,8 +13,10 @@ before_script:
   - apt-get update -yqq
   - apt-get install apt-transport-https -yqq
   # Add keyserver for SBT
-  - echo "deb http://dl.bintray.com/sbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
-  - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | tee -a /etc/apt/sources.list.d/sbt.list
+  - mkdir -p /root/.gnupg
+  - gpg --recv-keys --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --keyserver hkp://keyserver.ubuntu.com:80 2EE0EA64E40A89B84B2DF73499E82A75642AC823
+  - chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
   # Install SBT
   - apt-get update -yqq
   - apt-get install sbt -yqq
diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
index 544774d3b06..01041f4f056 100644
--- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml
@@ -11,11 +11,11 @@
 variables:
     FUZZAPI_VERSION: "1"
     SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
-    FUZZAPI_IMAGE: ${SECURE_ANALYZERS_PREFIX}/api-fuzzing:${FUZZAPI_VERSION}
+    FUZZAPI_IMAGE: api-fuzzing
 
 apifuzzer_fuzz:
     stage: fuzz
-    image: $FUZZAPI_IMAGE
+    image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION
     allow_failure: true
     rules:
         - if: $API_FUZZING_DISABLED
diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
index 89e6743b0e4..65a2b20d5c0 100644
--- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml
@@ -38,7 +38,8 @@ container_scanning:
   artifacts:
     reports:
       container_scanning: gl-container-scanning-report.json
-    paths: [gl-container-scanning-report.json]
+      dependency_scanning: gl-dependency-scanning-report.json
+    paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json]
   dependencies: []
   script:
     - gtcs scan
diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..57f1993921d
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml
@@ -0,0 +1,52 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Dast-API.gitlab-ci.yml
+
+# To use this template, add the following to your .gitlab-ci.yml file:
+#
+# include:
+#   template: DAST-API.latest.gitlab-ci.yml
+#
+# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST API:
+#
+# stages:
+#   - build
+#   - test
+#   - deploy
+#   - dast
+
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html
+
+# Configure DAST API scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
+# List of available variables: https://docs.gitlab.com/ee/user/application_security/dast_api/index.html#available-cicd-variables
+
+variables:
+  # Setting this variable affects all Security templates
+  # (SAST, Dependency Scanning, ...)
+  SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+  #
+  DAST_API_VERSION: "1"
+  DAST_API_IMAGE: api-fuzzing
+
+dast_api:
+  stage: dast
+  image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION
+  allow_failure: true
+  rules:
+    - if: $DAST_API_DISABLED
+      when: never
+    - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH &&
+          $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+      when: never
+    - if: $CI_COMMIT_BRANCH
+  script:
+    - /peach/analyzer-dast-api
+  artifacts:
+    when: always
+    paths:
+      - gl-assets
+      - gl-dast-api-report.json
+      - gl-*.log
+    reports:
+      dast: gl-dast-api-report.json
diff --git a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
index 4f63ff93d4d..8f4a836441d 100644
--- a/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Verify/Accessibility.gitlab-ci.yml
@@ -13,7 +13,7 @@ stages:
 
 a11y:
   stage: accessibility
-  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.0.1
+  image: registry.gitlab.com/gitlab-org/ci-cd/accessibility:6.1.1
   script: /gitlab-accessibility.sh $a11y_urls
   allow_failure: true
   artifacts:
diff --git a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml
index edd0fb0ba07..09fce67db2d 100644
--- a/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/dotNET-Core.gitlab-ci.yml
@@ -1,4 +1,4 @@
-# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
 # https://docs.gitlab.com/ee/development/cicd/templates.html
 # This specific template is located at:
 # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/dotNET-Core.yml
@@ -14,10 +14,10 @@
 # The 'latest' tag targets the latest available version of .NET Core SDK image.
 # If preferred, you can explicitly specify version of .NET Core (e.g. using '2.2-sdk' tag).
 #
-# See other available tags for .NET Core: https://hub.docker.com/r/microsoft/dotnet
+# See other available tags for .NET Core: https://hub.docker.com/_/microsoft-dotnet
 # Learn more about Docker tags: https://docs.docker.com/glossary/?term=tag
 # and the Docker itself: https://opensource.com/resources/what-docker
-image: microsoft/dotnet:latest
+image: mcr.microsoft.com/dotnet/sdk:latest
 
 # ### Define variables
 #
diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb
index f4c5a06af97..3e2c2c7fc1a 100644
--- a/lib/gitlab/ci/variables/builder.rb
+++ b/lib/gitlab/ci/variables/builder.rb
@@ -12,7 +12,7 @@ module Gitlab
 
         def scoped_variables(job, environment:, dependencies:)
           Gitlab::Ci::Variables::Collection.new.tap do |variables|
-            variables.concat(predefined_variables(job)) if pipeline.predefined_vars_in_builder_enabled?
+            variables.concat(predefined_variables(job))
           end
         end
 
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index 1aa3dbc5e47..296b0cfded2 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -29,10 +29,8 @@ module Gitlab
         run_logical_validations!
 
         Result.new(ci_config: @ci_config, warnings: @ci_config&.warnings)
-
       rescue Gitlab::Ci::Config::ConfigError => e
         Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings)
-
       rescue ValidationError => e
         Result.new(ci_config: @ci_config, errors: [e.message], warnings: @ci_config&.warnings)
       end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index 6215ba40ebe..f14279dca2d 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -92,6 +92,7 @@ module Gitlab
               script: job[:script],
               after_script: job[:after_script],
               environment: job[:environment],
+              resource_group_key: job[:resource_group],
               retry: job[:retry],
               parallel: job[:parallel],
               instance: job[:instance],