Add latest changes from gitlab-org/gitlab@13-8-stable-eev13.8.0-rc42

36 files changed, 618 insertions, 47 deletions

diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 071a8ef830f..8ed4dc61920 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -70,6 +70,10 @@ module Gitlab
         @normalized_jobs ||= Ci::Config::Normalizer.new(jobs).normalize_jobs
       end
 
+      def included_templates
+        @context.expandset.filter_map { |i| i[:template] }
+      end
+
       private
 
       def expand_config(config)
@@ -98,7 +102,8 @@ module Gitlab
           project: project,
           sha: sha || project&.repository&.root_ref_sha,
           user: user,
-          parent_pipeline: parent_pipeline)
+          parent_pipeline: parent_pipeline,
+          variables: project&.predefined_variables&.to_runner_variables)
       end
 
       def track_and_raise_for_dev_exception(error)
diff --git a/lib/gitlab/ci/config/entry/artifacts.rb b/lib/gitlab/ci/config/entry/artifacts.rb
index 206dbaea272..6118ff49928 100644
--- a/lib/gitlab/ci/config/entry/artifacts.rb
+++ b/lib/gitlab/ci/config/entry/artifacts.rb
@@ -12,7 +12,7 @@ module Gitlab
           include ::Gitlab::Config::Entry::Validatable
           include ::Gitlab::Config::Entry::Attributable
 
-          ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude].freeze
+          ALLOWED_KEYS = %i[name untracked paths reports when expire_in expose_as exclude public].freeze
           EXPOSE_AS_REGEX = /\A\w[-\w ]*\z/.freeze
           EXPOSE_AS_ERROR_MESSAGE = "can contain only letters, digits, '-', '_' and spaces"
 
@@ -27,6 +27,7 @@ module Gitlab
 
             with_options allow_nil: true do
               validates :name, type: String
+              validates :public, boolean: true
               validates :untracked, boolean: true
               validates :paths, array_of_strings: true
               validates :paths, array_of_strings: {
diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb
index cf6c2961ee7..e0adb1b19c2 100644
--- a/lib/gitlab/ci/config/external/context.rb
+++ b/lib/gitlab/ci/config/external/context.rb
@@ -7,14 +7,15 @@ module Gitlab
         class Context
           TimeoutError = Class.new(StandardError)
 
-          attr_reader :project, :sha, :user, :parent_pipeline
+          attr_reader :project, :sha, :user, :parent_pipeline, :variables
           attr_reader :expandset, :execution_deadline
 
-          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil)
+          def initialize(project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: [])
             @project = project
             @sha = sha
             @user = user
             @parent_pipeline = parent_pipeline
+            @variables = variables
             @expandset = Set.new
             @execution_deadline = 0
 
diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb
index e74f5b33de7..fdb3e1b00f9 100644
--- a/lib/gitlab/ci/config/external/file/local.rb
+++ b/lib/gitlab/ci/config/external/file/local.rb
@@ -41,7 +41,8 @@ module Gitlab
                 project: context.project,
                 sha: context.sha,
                 user: context.user,
-                parent_pipeline: context.parent_pipeline
+                parent_pipeline: context.parent_pipeline,
+                variables: context.variables
               }
             end
           end
diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb
index be479741784..114d493381c 100644
--- a/lib/gitlab/ci/config/external/file/project.rb
+++ b/lib/gitlab/ci/config/external/file/project.rb
@@ -72,7 +72,8 @@ module Gitlab
                 project: project,
                 sha: sha,
                 user: context.user,
-                parent_pipeline: context.parent_pipeline
+                parent_pipeline: context.parent_pipeline,
+                variables: context.variables
               }
             end
           end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 90692eafc3f..4d91cfd4c57 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -34,6 +34,7 @@ module Gitlab
               .compact
               .map(&method(:normalize_location))
               .flat_map(&method(:expand_project_files))
+              .map(&method(:expand_variables))
               .each(&method(:verify_duplicates!))
               .map(&method(:select_first_matching))
           end
@@ -47,14 +48,14 @@ module Gitlab
           # convert location if String to canonical form
           def normalize_location(location)
             if location.is_a?(String)
-              normalize_location_string(location)
+              expanded_location = expand_variables(location)
+              normalize_location_string(expanded_location)
             else
               location.deep_symbolize_keys
             end
           end
 
           def expand_project_files(location)
-            return location unless ::Feature.enabled?(:ci_include_multiple_files_from_project, context.project, default_enabled: true)
             return location unless location[:project]
 
             Array.wrap(location[:file]).map do |file|
@@ -96,6 +97,33 @@ module Gitlab
 
             matching.first
           end
+
+          def expand_variables(data)
+            return data unless ::Feature.enabled?(:variables_in_include_section_ci)
+
+            if data.is_a?(String)
+              expand(data)
+            else
+              transform(data)
+            end
+          end
+
+          def transform(data)
+            data.transform_values do |values|
+              case values
+              when Array
+                values.map { |value| expand(value.to_s) }
+              when String
+                expand(values)
+              else
+                values
+              end
+            end
+          end
+
+          def expand(data)
+            ExpandVariables.expand(data, context.variables)
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index af1df933b36..7956cf14203 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -56,23 +56,19 @@ module Gitlab
       end
 
       def self.pipeline_open_merge_requests?(project)
-        ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: false)
-      end
-
-      def self.seed_block_run_before_workflow_rules_enabled?(project)
-        ::Feature.enabled?(:ci_seed_block_run_before_workflow_rules, project, default_enabled: true)
+        ::Feature.enabled?(:ci_pipeline_open_merge_requests, project, default_enabled: true)
       end
 
       def self.ci_pipeline_editor_page_enabled?(project)
-        ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: false)
+        ::Feature.enabled?(:ci_pipeline_editor_page, project, default_enabled: :yaml)
       end
 
       def self.allow_failure_with_exit_codes_enabled?
-        ::Feature.enabled?(:ci_allow_failure_with_exit_codes)
+        ::Feature.enabled?(:ci_allow_failure_with_exit_codes, default_enabled: :yaml)
       end
 
       def self.rules_variables_enabled?(project)
-        ::Feature.enabled?(:ci_rules_variables, project, default_enabled: false)
+        ::Feature.enabled?(:ci_rules_variables, project, default_enabled: true)
       end
     end
   end
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index fb795152abe..364e67db02b 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -18,9 +18,10 @@ module Gitlab
         end
       end
 
-      def initialize(project:, current_user:)
+      def initialize(project:, current_user:, sha: nil)
         @project = project
         @current_user = current_user
+        @sha = sha || project.repository.commit.sha
       end
 
       def validate(content, dry_run: false)
@@ -51,7 +52,7 @@ module Gitlab
           content,
           project: @project,
           user: @current_user,
-          sha: @project.repository.commit.sha
+          sha: @sha
         ).execute
 
         Result.new(
@@ -99,7 +100,8 @@ module Gitlab
               except: job[:except],
               environment: job[:environment],
               when: job[:when],
-              allow_failure: job[:allow_failure]
+              allow_failure: job[:allow_failure],
+              needs: job.dig(:needs_attributes)
             }
           end
         end
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
index 57f73c265b2..985639982aa 100644
--- a/lib/gitlab/ci/parsers.rb
+++ b/lib/gitlab/ci/parsers.rb
@@ -15,8 +15,8 @@ module Gitlab
         }
       end
 
-      def self.fabricate!(file_type)
-        parsers.fetch(file_type.to_sym).new
+      def self.fabricate!(file_type, *args)
+        parsers.fetch(file_type.to_sym).new(*args)
       rescue KeyError
         raise ParserNotFoundError, "Cannot find any parser matching file type '#{file_type}'"
       end
diff --git a/lib/gitlab/ci/parsers/coverage/cobertura.rb b/lib/gitlab/ci/parsers/coverage/cobertura.rb
index 1edcbac2f25..eb3adf713d4 100644
--- a/lib/gitlab/ci/parsers/coverage/cobertura.rb
+++ b/lib/gitlab/ci/parsers/coverage/cobertura.rb
@@ -36,7 +36,7 @@ module Gitlab
           end
 
           def parse_node(key, value, coverage_report, context)
-            if key == 'sources' && value['source'].present?
+            if key == 'sources' && value && value['source'].present?
               parse_sources(value['source'], context)
             elsif key == 'package'
               Array.wrap(value).each do |item|
diff --git a/lib/gitlab/ci/pipeline/chain/build.rb b/lib/gitlab/ci/pipeline/chain/build.rb
index 9662209f88e..f0548284001 100644
--- a/lib/gitlab/ci/pipeline/chain/build.rb
+++ b/lib/gitlab/ci/pipeline/chain/build.rb
@@ -5,6 +5,9 @@ module Gitlab
     module Pipeline
       module Chain
         class Build < Chain::Base
+          include Gitlab::Allowable
+          include Chain::Helpers
+
           def perform!
             @pipeline.assign_attributes(
               source: @command.source,
@@ -20,12 +23,34 @@ module Gitlab
               pipeline_schedule: @command.schedule,
               merge_request: @command.merge_request,
               external_pull_request: @command.external_pull_request,
-              variables_attributes: Array(@command.variables_attributes)
+              locked: @command.project.latest_pipeline_locked,
+              variables_attributes: variables_attributes
             )
           end
 
           def break?
-            false
+            @pipeline.errors.any?
+          end
+
+          private
+
+          def variables_attributes
+            variables = Array(@command.variables_attributes)
+
+            # We allow parent pipelines to pass variables to child pipelines since
+            # these variables are coming from internal configurations. We will check
+            # permissions to :set_pipeline_variables when those are injected upstream,
+            # to the parent pipeline.
+            # In other scenarios (e.g. multi-project pipelines or run pipeline via UI)
+            # the variables are provided from the outside and those should be guarded.
+            return variables if @command.creates_child_pipeline?
+
+            if variables.present? && !can?(@command.current_user, :set_pipeline_variables, @command.project)
+              error("Insufficient permissions to set pipeline variables")
+              variables = []
+            end
+
+            variables
           end
         end
       end
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index d05be54267c..815fe6bac6d 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -79,6 +79,10 @@ module Gitlab
             bridge&.parent_pipeline
           end
 
+          def creates_child_pipeline?
+            bridge&.triggers_child_pipeline?
+          end
+
           def metrics
             @metrics ||= ::Gitlab::Ci::Pipeline::Metrics.new
           end
diff --git a/lib/gitlab/ci/pipeline/chain/seed.rb b/lib/gitlab/ci/pipeline/chain/seed.rb
index 083f0bec1df..7b537125b9b 100644
--- a/lib/gitlab/ci/pipeline/chain/seed.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed.rb
@@ -19,13 +19,6 @@ module Gitlab
             # Build to prevent erroring out on ambiguous refs.
             pipeline.protected = @command.protected_ref?
 
-            unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
-              ##
-              # Populate pipeline with block argument of CreatePipelineService#execute.
-              #
-              @command.seeds_block&.call(pipeline)
-            end
-
             ##
             # Gather all runtime build/stage errors
             #
diff --git a/lib/gitlab/ci/pipeline/chain/seed_block.rb b/lib/gitlab/ci/pipeline/chain/seed_block.rb
index f8e62949bea..67424635603 100644
--- a/lib/gitlab/ci/pipeline/chain/seed_block.rb
+++ b/lib/gitlab/ci/pipeline/chain/seed_block.rb
@@ -9,8 +9,6 @@ module Gitlab
           include Gitlab::Utils::StrongMemoize
 
           def perform!
-            return unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
-
             ##
             # Populate pipeline with block argument of CreatePipelineService#execute.
             #
@@ -20,8 +18,6 @@ module Gitlab
           end
 
           def break?
-            return false unless ::Gitlab::Ci::Features.seed_block_run_before_workflow_rules_enabled?(project)
-
             pipeline.errors.any?
           end
         end
diff --git a/lib/gitlab/ci/pipeline/chain/template_usage.rb b/lib/gitlab/ci/pipeline/chain/template_usage.rb
new file mode 100644
index 00000000000..c1a7b4ed453
--- /dev/null
+++ b/lib/gitlab/ci/pipeline/chain/template_usage.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Pipeline
+      module Chain
+        class TemplateUsage < Chain::Base
+          def perform!
+            included_templates.each do |template|
+              track_event(template)
+            end
+          end
+
+          def break?
+            false
+          end
+
+          private
+
+          def track_event(template)
+            Gitlab::UsageDataCounters::CiTemplateUniqueCounter
+              .track_unique_project_event(project_id: pipeline.project_id, template: template)
+          end
+
+          def included_templates
+            command.yaml_processor_result.included_templates
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
index 8f1e690c081..e68d9020a21 100644
--- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
+++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb
@@ -19,7 +19,7 @@ module Gitlab
               end
 
               unless allowed_to_write_ref?
-                error("Insufficient permissions for protected ref '#{command.ref}'")
+                error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance. <a href=https://docs.gitlab.com/ee/ci/pipelines/#pipeline-security-on-protected-branches>Learn more</a>".html_safe)
               end
             end
 
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 2271915a72b..fe3c2bca551 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -134,7 +134,7 @@ module Gitlab
                 stage.seeds_names.include?(need[:name])
               end
 
-              "#{name}: needs '#{need[:name]}'" unless result
+              "'#{name}' job needs '#{need[:name]}' job, but it was not added to the pipeline" unless result
             end.compact
           end
 
diff --git a/lib/gitlab/ci/reports/test_failure_history.rb b/lib/gitlab/ci/reports/test_failure_history.rb
index beceac5423a..c024e794ad5 100644
--- a/lib/gitlab/ci/reports/test_failure_history.rb
+++ b/lib/gitlab/ci/reports/test_failure_history.rb
@@ -12,8 +12,6 @@ module Gitlab
         end
 
         def load!
-          return unless Feature.enabled?(:test_failure_history, project)
-
           recent_failures_count.each do |key_hash, count|
             failed_test_cases[key_hash].set_recent_failures(count, project.default_branch_or_master)
           end
diff --git a/lib/gitlab/ci/status/group/factory.rb b/lib/gitlab/ci/status/group/factory.rb
index ee785856fdd..37e2b7320e2 100644
--- a/lib/gitlab/ci/status/group/factory.rb
+++ b/lib/gitlab/ci/status/group/factory.rb
@@ -8,6 +8,10 @@ module Gitlab
           def self.common_helpers
             Status::Group::Common
           end
+
+          def self.extended_statuses
+            [[Status::SuccessWarning]]
+          end
         end
       end
     end
diff --git a/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml
new file mode 100644
index 00000000000..7182b96594d
--- /dev/null
+++ b/lib/gitlab/ci/syntax_templates/Artifacts example.gitlab-ci.yml
@@ -0,0 +1,52 @@
+#
+# You can use artifacts to pass data to jobs in later stages.
+# For more information, see https://docs.gitlab.com/ee/ci/pipelines/job_artifacts.html
+#
+
+stages:
+  - build
+  - test
+  - deploy
+
+build-job:
+  stage: build
+  script:
+    - echo "This job might build an important file, and pass it to later jobs."
+    - echo "This is the content of the important file" > important-file.txt
+  artifacts:
+    paths:
+      - important-file.txt
+
+test-job-with-artifacts:
+  stage: test
+  script:
+    - echo "This job uses the artifact from the job in the earlier stage."
+    - cat important-file.txt
+    - echo "It creates another file, and adds it to the artifacts."
+    - echo "This is a second important file" > important-file2.txt
+  artifacts:
+    paths:
+      - important-file2.txt
+
+test-job-with-no-artifacts:
+  stage: test
+  dependencies: [] # Use to skip downloading any artifacts
+  script:
+    - echo "This job does not get the artifacts from other jobs."
+    - cat important-file.txt || exit 0
+
+deploy-job-with-all-artifacts:
+  stage: deploy
+  script:
+    - echo "By default, jobs download all available artifacts."
+    - cat important-file.txt
+    - cat important-file2.txt
+
+deploy-job-with-1-artifact:
+  stage: deploy
+  dependencies:
+    - build-job # Download artifacts from only this job
+  script:
+    - echo "You can configure a job to download artifacts from only certain jobs."
+    - cat important-file.txt
+    - cat important-file2.txt || exit 0
diff --git a/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml
new file mode 100644
index 00000000000..382bac09ed7
--- /dev/null
+++ b/lib/gitlab/ci/syntax_templates/Before_script and after_script example.gitlab-ci.yml
@@ -0,0 +1,36 @@
+#
+# You can define common tasks and run them before or after the main scripts in jobs.
+# For more information, see:
+#   - https://docs.gitlab.com/ee/ci/yaml/README.html#before_script
+#   - https://docs.gitlab.com/ee/ci/yaml/README.html#after_script
+#
+
+stages:
+  - test
+
+default:
+  before_script:
+    - echo "This script runs before the main script in every job, unless the job overrides it."
+    - echo "It may set up common dependencies, for example."
+  after_script:
+    - echo "This script runs after the main script in every job, unless the job overrides it."
+    - echo "It may do some common final clean up tasks"
+
+job-standard:
+  stage: test
+  script:
+    - echo "This job uses both of the globally defined before and after scripts."
+
+job-override-before:
+  stage: test
+  before_script:
+    - echo "Use a different before_script in this job."
+  script:
+    - echo "This job uses its own before_script, and the global after_script."
+
+job-override-after:
+  stage: test
+  after_script:
+    - echo "Use a different after_script in this job."
+  script:
+    - echo "This job uses its own after_script, and the global before_script."
diff --git a/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml
new file mode 100644
index 00000000000..5f27def74c9
--- /dev/null
+++ b/lib/gitlab/ci/syntax_templates/Manual jobs example.gitlab-ci.yml
@@ -0,0 +1,53 @@
+#
+# A manual job is a type of job that is not executed automatically and must be explicitly started by a user.
+# To make a job manual, add when: manual to its configuration.
+# For more information, see https://docs.gitlab.com/ee/ci/yaml/README.html#whenmanual
+#
+
+stages:
+  - build
+  - test
+  - deploy
+
+build-job:
+  stage: build
+  script:
+    - echo "This job is not a manual job"
+
+manual-build:
+  stage: build
+  script:
+    - echo "This manual job passes after you trigger it."
+  when: manual
+
+manual-build-allowed-to-fail:
+  stage: build
+  script:
+    - echo "This manual job fails after you trigger it."
+    - echo "It is allowed to fail, so the pipeline does not fail.
+  when: manual
+  allow_failure: true # Default behavior
+
+test-job:
+  stage: test
+  script:
+    - echo "This is a normal test job"
+    - echo "It runs when the when the build stage completes."
+    - echo "It does not need to wait for the manual jobs in the build stage to run."
+
+manual-test-not-allowed-to-fail:
+  stage: test
+  script:
+    - echo "This manual job fails after you trigger it."
+    - echo "It is NOT allowed to fail, so the pipeline is marked as failed
+    - echo "when this job completes."
+    - exit 1
+  when: manual
+  allow_failure: false # Optional behavior
+
+deploy-job:
+  stage: deploy
+  script:
+    - echo "This is a normal deploy job"
+    - echo "If a manual job that isn't allowed to fail ran in an earlier stage and failed,
+    - echo "this job does not run".
diff --git a/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml
new file mode 100644
index 00000000000..aced628aacb
--- /dev/null
+++ b/lib/gitlab/ci/syntax_templates/Multi-stage pipeline example.gitlab-ci.yml
@@ -0,0 +1,33 @@
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages
+#
+
+stages:
+  - build
+  - test
+  - deploy
+
+build-job:
+  stage: build
+  script:
+    - echo "This job runs in the build stage, which runs first."
+
+test-job1:
+  stage: test
+  script:
+    - echo "This job runs in the test stage."
+    - echo "It only starts when the job in the build stage completes successfully."
+
+test-job2:
+  stage: test
+  script:
+    - echo "This job also runs in the test stage."
+    - echo "This job can run at the same time as test-job2."
+
+deploy-job:
+  stage: deploy
+  script:
+    - echo "This job runs in the deploy stage."
+    - echo "It only runs when both jobs in the test stage complete successfully"
diff --git a/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml
new file mode 100644
index 00000000000..2b8cf7bab44
--- /dev/null
+++ b/lib/gitlab/ci/syntax_templates/Variables example.gitlab-ci.yml
@@ -0,0 +1,47 @@
+#
+# Variables can be used to for more dynamic behavior in jobs and scripts.
+# For more information, see https://docs.gitlab.com/ee/ci/variables/README.html
+#
+
+stages:
+  - test
+
+variables:
+  VAR1: "Variable 1 defined globally"
+
+use-a-variable:
+  stage: test
+  script:
+    - echo "You can use variables in jobs."
+    - echo "The content of 'VAR1' is = $VAR1"
+
+override-a-variable:
+  stage: test
+  variables:
+    VAR1: "Variable 1 was overriden in in the job."
+  script:
+    - echo "You can override global variables in jobs."
+    - echo "The content of 'VAR1' is = $VAR1"
+
+define-a-new-variable:
+  stage: test
+  variables:
+    VAR2: "Variable 2 is new and defined in the job only."
+  script:
+    - echo "You can mix global variables with variables defined in jobs."
+    - echo "The content of 'VAR1' is = $VAR1"
+    - echo "The content of 'VAR2' is = $VAR2"
+
+incorrect-variable-usage:
+  stage: test
+  script:
+    - echo "You can't use variables only defined in other jobs."
+    - echo "The content of 'VAR2' is = $VAR2"
+
+predefined-variables:
+  stage: test
+  script:
+    - echo "Some variables are predefined by GitLab CI/CD, for example:"
+    - echo "The commit author's username is $GITLAB_USER_LOGIN"
+    - echo "The commit branch is $CI_COMMIT_BRANCH"
+    - echo "The project path is $CI_PROJECT_PATH"
diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
new file mode 100644
index 00000000000..c06ef83c180
--- /dev/null
+++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
@@ -0,0 +1,84 @@
+# This template is on early stage of development.
+# Use it with caution. For usage instruction please read
+# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md
+
+include:
+  # workflow rules to prevent duplicate detached pipelines
+  - template: 'Workflows/Branch-Pipelines.gitlab-ci.yml'
+  # auto devops build
+  - template: 'Jobs/Build.gitlab-ci.yml'
+
+stages:
+  - build
+  - test
+  - provision
+  - deploy
+  - destroy
+
+variables:
+  TF_ADDRESS: ${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/terraform/state/${CI_COMMIT_REF_SLUG}
+  TF_VAR_ENVIRONMENT_NAME: ${CI_PROJECT_PATH_SLUG}_${CI_PROJECT_ID}_${CI_COMMIT_REF_SLUG}
+  TF_VAR_SERVICE_DESK_EMAIL: incoming+${CI_PROJECT_PATH_SLUG}-${CI_PROJECT_ID}-issue-@incoming.gitlab.com
+  TF_VAR_SHORT_ENVIRONMENT_NAME: ${CI_PROJECT_ID}-${CI_COMMIT_REF_SLUG}
+  TF_VAR_SMTP_FROM: ${SMTP_FROM}
+
+cache:
+  paths:
+    - .terraform
+
+.needs_aws_vars:
+  rules:
+    - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION'
+      when: on_success
+    - when: never
+
+terraform_apply:
+  stage: provision
+  image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable
+  extends: .needs_aws_vars
+  resource_group: terraform
+  before_script:
+    - cp /*.tf .
+    - cp /deploy.sh .
+  script:
+    - gitlab-terraform init
+    - gitlab-terraform plan
+    - gitlab-terraform plan-json
+    - gitlab-terraform apply
+
+deploy:
+  stage: deploy
+  image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable
+  extends: .needs_aws_vars
+  resource_group: deploy
+  before_script:
+    - cp /*.tf .
+    - cp /deploy.sh .
+    - cp /conf.nginx .
+  script:
+    - ./deploy.sh
+  artifacts:
+    reports:
+      dotenv: deploy.env
+  environment:
+    name: $CI_COMMIT_REF_SLUG
+    url: $DYNAMIC_ENVIRONMENT_URL
+    on_stop: terraform_destroy
+
+terraform_destroy:
+  variables:
+    GIT_STRATEGY: none
+  stage: destroy
+  image: registry.gitlab.com/gitlab-org/5-minute-production-app/deploy-template/stable
+  before_script:
+    - cp /*.tf .
+    - cp /deploy.sh .
+  script:
+    - gitlab-terraform destroy -auto-approve
+  environment:
+    name: $CI_COMMIT_REF_SLUG
+    action: stop
+  rules:
+    - if: '$AWS_ACCESS_KEY_ID && $AWS_SECRET_ACCESS_KEY && $AWS_DEFAULT_REGION && $CI_COMMIT_REF_PROTECTED == "false"'
+      when: manual
+    - when: never
diff --git a/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml
new file mode 100644
index 00000000000..504ece611ca
--- /dev/null
+++ b/lib/gitlab/ci/templates/Flutter.gitlab-ci.yml
@@ -0,0 +1,29 @@
+code_quality:
+  stage: test
+  image: "cirrusci/flutter:1.22.5"
+  before_script:
+    - pub global activate dart_code_metrics
+    - export PATH="$PATH":"$HOME/.pub-cache/bin"
+  script:
+    - metrics lib -r codeclimate  > gl-code-quality-report.json
+  artifacts:
+    reports:
+      codequality: gl-code-quality-report.json
+
+test:
+  stage: test
+  image: "cirrusci/flutter:1.22.5"
+  before_script:
+    - pub global activate junitreport
+    - export PATH="$PATH":"$HOME/.pub-cache/bin"
+  script:
+    - flutter test --machine --coverage | tojunit -o report.xml
+    - lcov --summary coverage/lcov.info
+    - genhtml coverage/lcov.info --output=coverage
+  coverage: '/lines\.*: \d+\.\d+\%/'
+  artifacts:
+    name: coverage
+    paths:
+      - $CI_PROJECT_DIR/coverage
+    reports:
+      junit: report.xml
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 2ae9730ec1a..501d8737acd 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -7,7 +7,7 @@ code_quality:
   variables:
     DOCKER_DRIVER: overlay2
     DOCKER_TLS_CERTDIR: ""
-    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.18-gitlab.1"
+    CODE_QUALITY_IMAGE: "registry.gitlab.com/gitlab-org/ci-cd/codequality:0.85.19"
   needs: []
   script:
     - export SOURCE_CODE=$PWD
diff --git a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
index 23dfeda31cc..192b1509fdc 100644
--- a/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Managed-Cluster-Applications.gitlab-ci.yml
@@ -1,6 +1,6 @@
 apply:
   stage: deploy
-  image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.36.0"
+  image: "registry.gitlab.com/gitlab-org/cluster-integration/cluster-applications:v0.37.0"
   environment:
     name: production
   variables:
diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
new file mode 100644
index 00000000000..fc1acd09714
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml
@@ -0,0 +1,43 @@
+# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dast/
+
+# Configure the scanning tool through the environment variables.
+# List of the variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables
+# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
+
+variables:
+  DAST_VERSION: 1
+  # Setting this variable will affect all Security templates
+  # (SAST, Dependency Scanning, ...)
+  SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
+
+dast:
+  stage: dast
+  image:
+    name: "$SECURE_ANALYZERS_PREFIX/dast:$DAST_VERSION"
+  variables:
+    GIT_STRATEGY: none
+  allow_failure: true
+  script:
+    - export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
+    - if [ -z "$DAST_WEBSITE$DAST_API_SPECIFICATION" ]; then echo "Either DAST_WEBSITE or DAST_API_SPECIFICATION must be set. See https://docs.gitlab.com/ee/user/application_security/dast/#configuration for more details." && exit 1; fi
+    - /analyze
+  artifacts:
+    reports:
+      dast: gl-dast-report.json
+  rules:
+    - if: $DAST_DISABLED
+      when: never
+    - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH &&
+          $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME
+      when: never
+    - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME &&
+          $REVIEW_DISABLED && $DAST_WEBSITE == null &&
+          $DAST_API_SPECIFICATION == null
+      when: never
+    - if: $CI_COMMIT_BRANCH &&
+          $CI_KUBERNETES_ACTIVE &&
+          $GITLAB_FEATURES =~ /\bdast\b/
+    - if: $CI_COMMIT_BRANCH &&
+          $DAST_WEBSITE
+    - if: $CI_COMMIT_BRANCH &&
+          $DAST_API_SPECIFICATION
diff --git a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
index f4ee8ebd47e..56c6fbd96bc 100644
--- a/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/SAST.gitlab-ci.yml
@@ -10,6 +10,7 @@ variables:
   SECURE_ANALYZERS_PREFIX: "registry.gitlab.com/gitlab-org/security-products/analyzers"
 
   SAST_DEFAULT_ANALYZERS: "bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, sobelow, pmd-apex, kubesec, mobsf"
+  SAST_EXCLUDED_ANALYZERS: ""
   SAST_EXCLUDED_PATHS: "spec, test, tests, tmp"
   SAST_ANALYZER_IMAGE_TAG: 2
   SCAN_KUBERNETES_MANIFESTS: "false"
@@ -44,6 +45,8 @@ bandit-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /bandit/
       exists:
@@ -58,6 +61,8 @@ brakeman-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /brakeman/
       exists:
@@ -72,6 +77,8 @@ eslint-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /eslint/
       exists:
@@ -90,6 +97,8 @@ flawfinder-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /flawfinder/
       exists:
@@ -105,6 +114,8 @@ kubesec-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /kubesec/ &&
           $SCAN_KUBERNETES_MANIFESTS == 'true'
@@ -118,6 +129,8 @@ gosec-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /gosec/
       exists:
@@ -136,6 +149,8 @@ mobsf-android-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
           $SAST_EXPERIMENTAL_FEATURES == 'true'
@@ -155,6 +170,8 @@ mobsf-ios-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
           $SAST_EXPERIMENTAL_FEATURES == 'true'
@@ -170,6 +187,8 @@ nodejs-scan-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /nodejs-scan/
       exists:
@@ -184,6 +203,8 @@ phpcs-security-audit-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /phpcs-security-audit/
       exists:
@@ -198,6 +219,8 @@ pmd-apex-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /pmd-apex/
       exists:
@@ -212,6 +235,8 @@ security-code-scan-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /security-code-scan/
       exists:
@@ -227,6 +252,8 @@ sobelow-sast:
   rules:
     - if: $SAST_DISABLED
       when: never
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/
+      when: never
     - if: $CI_COMMIT_BRANCH &&
           $SAST_DEFAULT_ANALYZERS =~ /sobelow/
       exists:
@@ -239,6 +266,8 @@ spotbugs-sast:
   variables:
     SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG"
   rules:
+    - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/
+      when: never
     - if: $SAST_DEFAULT_ANALYZERS =~ /mobsf/ &&
           $SAST_EXPERIMENTAL_FEATURES == 'true'
       exists:
diff --git a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
index 8ca1d2e08ba..d2a6fa06dd8 100644
--- a/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secret-Detection.gitlab-ci.yml
@@ -37,6 +37,7 @@ secret_detection:
       when: never
     - if: $CI_COMMIT_BRANCH && $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
   script:
+    - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
     - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
     - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
     - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index 377c72e8031..7e2828d010f 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -17,6 +17,7 @@ variables:
 cache:
   paths:
     - .terraform
+    - .terraform.lock.hcl
 
 before_script:
   - alias convert_report="jq -r '([.resource_changes[]?.change.actions?]|flatten)|{\"create\":(map(select(.==\"create\"))|length),\"update\":(map(select(.==\"update\"))|length),\"delete\":(map(select(.==\"delete\"))|length)}'"
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index 910e711f046..c2db0fc44f1 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -19,6 +19,7 @@ cache:
   key: "${TF_ROOT}"
   paths:
     - ${TF_ROOT}/.terraform/
+    - ${TF_ROOT}/.terraform.lock.hcl
 
 .init: &init
   stage: init
diff --git a/lib/gitlab/ci/variables/collection/sorted.rb b/lib/gitlab/ci/variables/collection/sorted.rb
new file mode 100644
index 00000000000..6abc6a5644f
--- /dev/null
+++ b/lib/gitlab/ci/variables/collection/sorted.rb
@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Gitlab
+  module Ci
+    module Variables
+      class Collection
+        class Sorted
+          include TSort
+          include Gitlab::Utils::StrongMemoize
+
+          def initialize(variables)
+            @variables = variables
+          end
+
+          def valid?
+            errors.nil?
+          end
+
+          # errors sorts an array of variables, ignoring unknown variable references,
+          # and returning an error string if a circular variable reference is found
+          def errors
+            return if Feature.disabled?(:variable_inside_variable)
+
+            strong_memoize(:errors) do
+              # Check for cyclic dependencies and build error message in that case
+              errors = each_strongly_connected_component.filter_map do |component|
+                component.map { |v| v[:key] }.inspect if component.size > 1
+              end
+
+              "circular variable reference detected: #{errors.join(', ')}" if errors.any?
+            end
+          end
+
+          # sort sorts an array of variables, ignoring unknown variable references.
+          # If a circular variable reference is found, the original array is returned
+          def sort
+            return @variables if Feature.disabled?(:variable_inside_variable)
+            return @variables if errors
+
+            tsort
+          end
+
+          private
+
+          def tsort_each_node(&block)
+            @variables.each(&block)
+          end
+
+          def tsort_each_child(variable, &block)
+            each_variable_reference(variable[:value], &block)
+          end
+
+          def input_vars
+            strong_memoize(:input_vars) do
+              @variables.index_by { |env| env.fetch(:key) }
+            end
+          end
+
+          def walk_references(value)
+            return unless ExpandVariables.possible_var_reference?(value)
+
+            value.scan(ExpandVariables::VARIABLES_REGEXP) do |var_ref|
+              yield(input_vars, var_ref.first)
+            end
+          end
+
+          def each_variable_reference(value)
+            walk_references(value) do |vars_hash, ref_var_name|
+              variable = vars_hash.dig(ref_var_name)
+              yield variable if variable
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb
index ee55eb8b22a..dc4951f76bb 100644
--- a/lib/gitlab/ci/yaml_processor.rb
+++ b/lib/gitlab/ci/yaml_processor.rb
@@ -10,12 +10,6 @@ module Gitlab
     class YamlProcessor
       ValidationError = Class.new(StandardError)
 
-      def self.validation_message(content, opts = {})
-        result = new(content, opts).execute
-
-        result.errors.first
-      end
-
       def initialize(config_content, opts = {})
         @config_content = config_content
         @opts = opts
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index cd7d781a574..86749cda9c7 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -53,6 +53,10 @@ module Gitlab
           @stages ||= @ci_config.stages
         end
 
+        def included_templates
+          @included_templates ||= @ci_config.included_templates
+        end
+
         def build_attributes(name)
           job = jobs.fetch(name.to_sym, {})