Welcome to mirror list, hosted at ThFree Co, Russian Federation.

gitlab.com/gitlab-org/gitlab-foss.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorGitLab Bot <gitlab-bot@gitlab.com>2021-08-19 12:08:42 +0300
committerGitLab Bot <gitlab-bot@gitlab.com>2021-08-19 12:08:42 +0300
commitb76ae638462ab0f673e5915986070518dd3f9ad3 (patch)
treebdab0533383b52873be0ec0eb4d3c66598ff8b91 /lib/gitlab/ci
parent434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff)
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
-rw-r--r--lib/gitlab/ci/ansi2html.rb3
-rw-r--r--lib/gitlab/ci/ansi2json/line.rb9
-rw-r--r--lib/gitlab/ci/config.rb36
-rw-r--r--lib/gitlab/ci/config/entry/include.rb18
-rw-r--r--lib/gitlab/ci/config/entry/include/rules.rb28
-rw-r--r--lib/gitlab/ci/config/entry/include/rules/rule.rb30
-rw-r--r--lib/gitlab/ci/config/entry/inherit/variables.rb11
-rw-r--r--lib/gitlab/ci/config/entry/job.rb3
-rw-r--r--lib/gitlab/ci/config/entry/processable.rb21
-rw-r--r--lib/gitlab/ci/config/entry/rules.rb2
-rw-r--r--lib/gitlab/ci/config/external/file/remote.rb2
-rw-r--r--lib/gitlab/ci/config/external/mapper.rb12
-rw-r--r--lib/gitlab/ci/config/external/rules.rb31
-rw-r--r--lib/gitlab/ci/config/normalizer/matrix_strategy.rb1
-rw-r--r--lib/gitlab/ci/features.rb2
-rw-r--r--lib/gitlab/ci/limit.rb11
-rw-r--r--lib/gitlab/ci/lint.rb1
-rw-r--r--lib/gitlab/ci/model.rb15
-rw-r--r--lib/gitlab/ci/parsers.rb4
-rw-r--r--lib/gitlab/ci/parsers/security/common.rb266
-rw-r--r--lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb36
-rw-r--r--lib/gitlab/ci/parsers/security/sast.rb26
-rw-r--r--lib/gitlab/ci/parsers/security/secret_detection.rb27
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schema_validator.rb68
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/sast.json706
-rw-r--r--lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json729
-rw-r--r--lib/gitlab/ci/pipeline/chain/command.rb9
-rw-r--r--lib/gitlab/ci/pipeline/chain/config/process.rb2
-rw-r--r--lib/gitlab/ci/pipeline/chain/sequence.rb1
-rw-r--r--lib/gitlab/ci/pipeline/chain/skip.rb8
-rw-r--r--lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb2
-rw-r--r--lib/gitlab/ci/pipeline/metrics.rb11
-rw-r--r--lib/gitlab/ci/pipeline/seed/build.rb4
-rw-r--r--lib/gitlab/ci/reports/security/aggregated_report.rb24
-rw-r--r--lib/gitlab/ci/reports/security/finding.rb150
-rw-r--r--lib/gitlab/ci/reports/security/finding_key.rb36
-rw-r--r--lib/gitlab/ci/reports/security/finding_signature.rb46
-rw-r--r--lib/gitlab/ci/reports/security/locations/base.rb41
-rw-r--r--lib/gitlab/ci/reports/security/locations/sast.rb33
-rw-r--r--lib/gitlab/ci/reports/security/locations/secret_detection.rb33
-rw-r--r--lib/gitlab/ci/reports/security/report.rb76
-rw-r--r--lib/gitlab/ci/reports/security/reports.rb42
-rw-r--r--lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb163
-rw-r--r--lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Bash.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Django.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml1
-rw-r--r--lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml4
-rw-r--r--lib/gitlab/ci/templates/Laravel.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Ruby.gitlab-ci.yml3
-rw-r--r--lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml23
-rw-r--r--lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml12
-rw-r--r--lib/gitlab/ci/templates/Terraform.gitlab-ci.yml2
-rw-r--r--lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml15
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml64
-rw-r--r--lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml16
-rw-r--r--lib/gitlab/ci/yaml_processor/dag.rb2
-rw-r--r--lib/gitlab/ci/yaml_processor/result.rb2
65 files changed, 2835 insertions, 110 deletions
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb
index 97988d8aa13..ef936581c10 100644
--- a/lib/gitlab/ci/ansi2html.rb
+++ b/lib/gitlab/ci/ansi2html.rb
@@ -33,6 +33,8 @@ module Gitlab
Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride
class Converter
+ include EncodingHelper
+
def on_0(_)
reset
end
@@ -256,6 +258,7 @@ module Gitlab
start_offset = @offset
stream.each_line do |line|
+ line = encode_utf8_no_detect(line)
s = StringScanner.new(line)
until s.eos?
diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb
index 8f2d47e7ccc..e48080993ab 100644
--- a/lib/gitlab/ci/ansi2json/line.rb
+++ b/lib/gitlab/ci/ansi2json/line.rb
@@ -9,6 +9,8 @@ module Gitlab
# Line::Segment is a portion of a line that has its own style
# and text. Multiple segments make the line content.
class Segment
+ include EncodingHelper
+
attr_accessor :text, :style
def initialize(style:)
@@ -21,11 +23,12 @@ module Gitlab
end
def to_h
- # Without force encoding to UTF-8 we could get an error
- # when serializing the Hash to JSON.
+ # Without forcing the encoding to UTF-8 and then replacing
+ # invalid UTF-8 sequences we can get an error when serializing
+ # the Hash to JSON.
# Encoding::UndefinedConversionError:
# "\xE2" from ASCII-8BIT to UTF-8
- { text: text.force_encoding('UTF-8') }.tap do |result|
+ { text: encode_utf8_no_detect(text) }.tap do |result|
result[:style] = style.to_s if style.set?
end
end
diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb
index 9c6428d701c..aceaf012f7e 100644
--- a/lib/gitlab/ci/config.rb
+++ b/lib/gitlab/ci/config.rb
@@ -17,13 +17,13 @@ module Gitlab
Config::Yaml::Tags::TagError
].freeze
- attr_reader :root, :context, :ref, :source
+ attr_reader :root, :context, :source_ref_path, :source
- def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil)
- @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline)
+ def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil)
+ @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path)
@context.set_deadline(TIMEOUT_SECONDS)
- @ref = ref
+ @source_ref_path = source_ref_path
@source = source
@config = expand_config(config)
@@ -108,13 +108,37 @@ module Gitlab
end
end
- def build_context(project:, sha:, user:, parent_pipeline:)
+ def build_context(project:, sha:, user:, parent_pipeline:, ref:)
Config::External::Context.new(
project: project,
sha: sha || find_sha(project),
user: user,
parent_pipeline: parent_pipeline,
- variables: project&.predefined_variables&.to_runner_variables)
+ variables: build_variables(project: project, ref: ref))
+ end
+
+ def build_variables(project:, ref:)
+ Gitlab::Ci::Variables::Collection.new.tap do |variables|
+ break variables unless project
+
+ # The order of the following lines is important as priority of CI variables is
+ # defined globally within GitLab.
+ #
+ # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence
+ variables.concat(project.predefined_variables)
+ variables.concat(pipeline_predefined_variables(ref: ref))
+ variables.concat(project.ci_instance_variables_for(ref: ref))
+ variables.concat(project.group.ci_variables_for(ref, project)) if project.group
+ variables.concat(project.ci_variables_for(ref: ref))
+ end
+ end
+
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables
+ # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines.
+ def pipeline_predefined_variables(ref:)
+ Gitlab::Ci::Variables::Collection.new.tap do |v|
+ v.append(key: 'CI_COMMIT_REF_NAME', value: ref)
+ end
end
def track_and_raise_for_dev_exception(error)
diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb
index ad0ed00aa6f..368d8f07f8d 100644
--- a/lib/gitlab/ci/config/entry/include.rb
+++ b/lib/gitlab/ci/config/entry/include.rb
@@ -9,8 +9,10 @@ module Gitlab
#
class Include < ::Gitlab::Config::Entry::Node
include ::Gitlab::Config::Entry::Validatable
+ include ::Gitlab::Config::Entry::Configurable
+ include ::Gitlab::Config::Entry::Attributable
- ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze
+ ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze
validations do
validates :config, hash_or_string: true
@@ -27,6 +29,20 @@ module Gitlab
errors.add(:config, "must specify the file where to fetch the config from")
end
end
+
+ with_options allow_nil: true do
+ validates :rules, array_of_hashes: true
+ end
+ end
+
+ entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules,
+ description: 'List of evaluable Rules to determine file inclusion.',
+ inherit: false
+
+ attributes :rules
+
+ def skip_config_hash_validation?
+ true
end
end
end
diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb
new file mode 100644
index 00000000000..8eaf9e35aaf
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/include/rules.rb
@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ class Include
+ class Rules < ::Gitlab::Config::Entry::ComposableArray
+ include ::Gitlab::Config::Entry::Validatable
+
+ validations do
+ validates :config, presence: true
+ validates :config, type: Array
+ end
+
+ def value
+ @config
+ end
+
+ def composable_class
+ Entry::Include::Rules::Rule
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb
new file mode 100644
index 00000000000..d3d0f098814
--- /dev/null
+++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module Entry
+ class Include
+ class Rules::Rule < ::Gitlab::Config::Entry::Node
+ include ::Gitlab::Config::Entry::Validatable
+ include ::Gitlab::Config::Entry::Attributable
+
+ ALLOWED_KEYS = %i[if].freeze
+
+ attributes :if
+
+ validations do
+ validates :config, presence: true
+ validates :config, type: { with: Hash }
+ validates :config, allowed_keys: ALLOWED_KEYS
+
+ with_options allow_nil: true do
+ validates :if, expression: true
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb
index aa68833bdb8..adef4d1636a 100644
--- a/lib/gitlab/ci/config/entry/inherit/variables.rb
+++ b/lib/gitlab/ci/config/entry/inherit/variables.rb
@@ -13,9 +13,6 @@ module Gitlab
strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) }
class BooleanStrategy < ::Gitlab::Config::Entry::Boolean
- def inherit?(_key)
- value
- end
end
class ArrayStrategy < ::Gitlab::Config::Entry::Node
@@ -25,20 +22,12 @@ module Gitlab
validates :config, type: Array
validates :config, array_of_strings: true
end
-
- def inherit?(key)
- value.include?(key.to_s)
- end
end
class UnknownStrategy < ::Gitlab::Config::Entry::Node
def errors
["#{location} should be a bool or array of strings"]
end
-
- def inherit?(key)
- false
- end
end
end
end
diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb
index e6d63969161..bd4d5f33689 100644
--- a/lib/gitlab/ci/config/entry/job.rb
+++ b/lib/gitlab/ci/config/entry/job.rb
@@ -16,11 +16,8 @@ module Gitlab
environment coverage retry parallel interruptible timeout
release dast_configuration secrets].freeze
- REQUIRED_BY_NEEDS = %i[stage].freeze
-
validations do
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
- validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs?
validates :script, presence: true
with_options allow_nil: true do
diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb
index 79dfb0eec1d..3543b5493bd 100644
--- a/lib/gitlab/ci/config/entry/processable.rb
+++ b/lib/gitlab/ci/config/entry/processable.rb
@@ -31,7 +31,7 @@ module Gitlab
with_options allow_nil: true do
validates :extends, array_of_strings_or_string: true
- validates :rules, array_of_hashes: true
+ validates :rules, nested_array_of_hashes: true
validates :resource_group, type: String
end
end
@@ -88,9 +88,6 @@ module Gitlab
validate_against_warnings
end
- # inherit root variables
- @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables
-
yield if block_given?
end
end
@@ -123,27 +120,13 @@ module Gitlab
stage: stage_value,
extends: extends,
rules: rules_value,
- variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
- job_variables: job_variables,
+ job_variables: variables_value.to_h,
root_variables_inheritance: root_variables_inheritance,
only: only_value,
except: except_value,
resource_group: resource_group }.compact
end
- def root_and_job_variables_value
- root_variables = @root_variables_value.to_h # rubocop:disable Gitlab/ModuleWithInstanceVariables
- root_variables = root_variables.select do |key, _|
- inherit_entry&.variables_entry&.inherit?(key)
- end
-
- root_variables.merge(variables_value.to_h)
- end
-
- def job_variables
- variables_value.to_h
- end
-
def root_variables_inheritance
inherit_entry&.variables_entry&.value
end
diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb
index bf74f995e80..53e52981471 100644
--- a/lib/gitlab/ci/config/entry/rules.rb
+++ b/lib/gitlab/ci/config/entry/rules.rb
@@ -13,7 +13,7 @@ module Gitlab
end
def value
- @config
+ [@config].flatten
end
def composable_class
diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb
index 567a86c47e5..4bd8e250d7a 100644
--- a/lib/gitlab/ci/config/external/file/remote.rb
+++ b/lib/gitlab/ci/config/external/file/remote.rb
@@ -45,7 +45,7 @@ module Gitlab
errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!")
end
- response.to_s if errors.none?
+ response.body if errors.none?
end
end
end
diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb
index 3216d4eaac4..97e4922b2a1 100644
--- a/lib/gitlab/ci/config/external/mapper.rb
+++ b/lib/gitlab/ci/config/external/mapper.rb
@@ -33,6 +33,7 @@ module Gitlab
locations
.compact
.map(&method(:normalize_location))
+ .filter_map(&method(:verify_rules))
.flat_map(&method(:expand_project_files))
.flat_map(&method(:expand_wildcard_paths))
.map(&method(:expand_variables))
@@ -56,6 +57,15 @@ module Gitlab
end
end
+ def verify_rules(location)
+ # Behaves like there is no `rules`
+ return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml)
+
+ return unless Rules.new(location[:rules]).evaluate(context).pass?
+
+ location
+ end
+
def expand_project_files(location)
return location unless location[:project]
@@ -65,8 +75,6 @@ module Gitlab
end
def expand_wildcard_paths(location)
- return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml)
-
# We only support local files for wildcard paths
return location unless location[:local] && location[:local].include?('*')
diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb
new file mode 100644
index 00000000000..5a788427172
--- /dev/null
+++ b/lib/gitlab/ci/config/external/rules.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ class Config
+ module External
+ class Rules
+ def initialize(rule_hashes)
+ @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes)
+ end
+
+ def evaluate(context)
+ Result.new(@rule_list.nil? || match_rule(context))
+ end
+
+ private
+
+ def match_rule(context)
+ @rule_list.find { |rule| rule.matches?(nil, context) }
+ end
+
+ Result = Struct.new(:result) do
+ def pass?
+ !!result
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
index 5cabbc86d3e..312f98f850a 100644
--- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
+++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb
@@ -43,7 +43,6 @@ module Gitlab
{
name: name,
instance: instance,
- variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
job_variables: variables,
parallel: { total: total }
}.compact
diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb
index d26a903c1f8..51051b0490f 100644
--- a/lib/gitlab/ci/features.rb
+++ b/lib/gitlab/ci/features.rb
@@ -3,7 +3,7 @@
module Gitlab
module Ci
##
- # Ci::Features is a class that aggregates all CI/CD feature flags in one place.
+ # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place.
#
module Features
# NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project`
diff --git a/lib/gitlab/ci/limit.rb b/lib/gitlab/ci/limit.rb
index c22a3c503d5..4f914388969 100644
--- a/lib/gitlab/ci/limit.rb
+++ b/lib/gitlab/ci/limit.rb
@@ -24,10 +24,13 @@ module Gitlab
end
def log_error!(extra_context = {})
- error = LimitExceededError.new(message)
- # TODO: change this to Gitlab::ErrorTracking.log_exception(error, extra_context)
- # https://gitlab.com/gitlab-org/gitlab/issues/32906
- ::Gitlab::ErrorTracking.track_exception(error, extra_context)
+ ::Gitlab::ErrorTracking.log_exception(limit_exceeded_error, extra_context)
+ end
+
+ protected
+
+ def limit_exceeded_error
+ LimitExceededError.new(message)
end
end
end
diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb
index 4a7c11ee26e..cd2c135dd7e 100644
--- a/lib/gitlab/ci/lint.rb
+++ b/lib/gitlab/ci/lint.rb
@@ -38,6 +38,7 @@ module Gitlab
pipeline = ::Ci::CreatePipelineService
.new(@project, @current_user, ref: @project.default_branch)
.execute(:push, dry_run: true, content: content)
+ .payload
Result.new(
jobs: dry_run_convert_to_jobs(pipeline.stages),
diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb
deleted file mode 100644
index 1625cb841b6..00000000000
--- a/lib/gitlab/ci/model.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Ci
- module Model
- def table_name_prefix
- "ci_"
- end
-
- def model_name
- @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize)
- end
- end
- end
-end
diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb
index 3469537a2e2..1223d664214 100644
--- a/lib/gitlab/ci/parsers.rb
+++ b/lib/gitlab/ci/parsers.rb
@@ -11,7 +11,9 @@ module Gitlab
cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura,
terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan,
accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y,
- codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate
+ codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate,
+ sast: ::Gitlab::Ci::Parsers::Security::Sast,
+ secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection
}
end
diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
new file mode 100644
index 00000000000..41acb4d5040
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Security
+ class Common
+ SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError)
+
+ def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false)
+ new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse!
+ end
+
+ def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false)
+ @json_data = json_data
+ @report = report
+ @validate = validate
+ @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
+ end
+
+ def parse!
+ return report_data unless valid?
+
+ raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash)
+
+ create_scanner
+ create_scan
+ create_analyzer
+ set_report_version
+
+ create_findings
+
+ report_data
+ rescue JSON::ParserError
+ raise SecurityReportParserError, 'JSON parsing failed'
+ rescue StandardError => e
+ Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e)
+ raise SecurityReportParserError, "#{report.type} security report parsing failed"
+ end
+
+ private
+
+ attr_reader :json_data, :report, :validate
+
+ def valid?
+ return true if !validate || schema_validator.valid?
+
+ schema_validator.errors.each { |error| report.add_error('Schema', error) }
+
+ false
+ end
+
+ def schema_validator
+ @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data)
+ end
+
+ def report_data
+ @report_data ||= Gitlab::Json.parse!(json_data)
+ end
+
+ def report_version
+ @report_version ||= report_data['version']
+ end
+
+ def top_level_scanner
+ @top_level_scanner ||= report_data.dig('scan', 'scanner')
+ end
+
+ def scan_data
+ @scan_data ||= report_data.dig('scan')
+ end
+
+ def analyzer_data
+ @analyzer_data ||= report_data.dig('scan', 'analyzer')
+ end
+
+ def tracking_data(data)
+ data['tracking']
+ end
+
+ def create_findings
+ if report_data["vulnerabilities"]
+ report_data["vulnerabilities"].each { |finding| create_finding(finding) }
+ end
+ end
+
+ def create_finding(data, remediations = [])
+ identifiers = create_identifiers(data['identifiers'])
+ links = create_links(data['links'])
+ location = create_location(data['location'] || {})
+ signatures = create_signatures(tracking_data(data))
+
+ if @vulnerability_finding_signatures_enabled && !signatures.empty?
+ # NOT the signature_sha - the compare key is hashed
+ # to create the project_fingerprint
+ highest_priority_signature = signatures.max_by(&:priority)
+ uuid = calculate_uuid_v5(identifiers.first, highest_priority_signature.signature_hex)
+ else
+ uuid = calculate_uuid_v5(identifiers.first, location&.fingerprint)
+ end
+
+ report.add_finding(
+ ::Gitlab::Ci::Reports::Security::Finding.new(
+ uuid: uuid,
+ report_type: report.type,
+ name: finding_name(data, identifiers, location),
+ compare_key: data['cve'] || '',
+ location: location,
+ severity: parse_severity_level(data['severity']),
+ confidence: parse_confidence_level(data['confidence']),
+ scanner: create_scanner(data['scanner']),
+ scan: report&.scan,
+ identifiers: identifiers,
+ links: links,
+ remediations: remediations,
+ raw_metadata: data.to_json,
+ metadata_version: report_version,
+ details: data['details'] || {},
+ signatures: signatures,
+ project_id: report.project_id,
+ vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled))
+ end
+
+ def create_signatures(tracking)
+ tracking ||= { 'items' => [] }
+
+ signature_algorithms = Hash.new { |hash, key| hash[key] = [] }
+
+ tracking['items'].each do |item|
+ next unless item.key?('signatures')
+
+ item['signatures'].each do |signature|
+ alg = signature['algorithm']
+ signature_algorithms[alg] << signature['value']
+ end
+ end
+
+ signature_algorithms.map do |algorithm, values|
+ value = values.join('|')
+ signature = ::Gitlab::Ci::Reports::Security::FindingSignature.new(
+ algorithm_type: algorithm,
+ signature_value: value
+ )
+
+ if signature.valid?
+ signature
+ else
+ e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}")
+ Gitlab::ErrorTracking.track_exception(e)
+ nil
+ end
+ end.compact
+ end
+
+ def create_scan
+ return unless scan_data.is_a?(Hash)
+
+ report.scan = ::Gitlab::Ci::Reports::Security::Scan.new(scan_data)
+ end
+
+ def set_report_version
+ report.version = report_version
+ end
+
+ def create_analyzer
+ return unless analyzer_data.is_a?(Hash)
+
+ params = {
+ id: analyzer_data.dig('id'),
+ name: analyzer_data.dig('name'),
+ version: analyzer_data.dig('version'),
+ vendor: analyzer_data.dig('vendor', 'name')
+ }
+
+ return unless params.values.all?
+
+ report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params)
+ end
+
+ def create_scanner(scanner_data = top_level_scanner)
+ return unless scanner_data.is_a?(Hash)
+
+ report.add_scanner(
+ ::Gitlab::Ci::Reports::Security::Scanner.new(
+ external_id: scanner_data['id'],
+ name: scanner_data['name'],
+ vendor: scanner_data.dig('vendor', 'name'),
+ version: scanner_data.dig('version')))
+ end
+
+ def create_identifiers(identifiers)
+ return [] unless identifiers.is_a?(Array)
+
+ identifiers.map { |identifier| create_identifier(identifier) }.compact
+ end
+
+ def create_identifier(identifier)
+ return unless identifier.is_a?(Hash)
+
+ report.add_identifier(
+ ::Gitlab::Ci::Reports::Security::Identifier.new(
+ external_type: identifier['type'],
+ external_id: identifier['value'],
+ name: identifier['name'],
+ url: identifier['url']))
+ end
+
+ def create_links(links)
+ return [] unless links.is_a?(Array)
+
+ links.map { |link| create_link(link) }.compact
+ end
+
+ def create_link(link)
+ return unless link.is_a?(Hash)
+
+ ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url'])
+ end
+
+ def parse_severity_level(input)
+ input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' }
+ end
+
+ def parse_confidence_level(input)
+ input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' }
+ end
+
+ def create_location(location_data)
+ raise NotImplementedError
+ end
+
+ def finding_name(data, identifiers, location)
+ return data['message'] if data['message'].present?
+ return data['name'] if data['name'].present?
+
+ identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first
+ "#{identifier.name} in #{location&.fingerprint_path}"
+ end
+
+ def calculate_uuid_v5(primary_identifier, location_fingerprint)
+ uuid_v5_name_components = {
+ report_type: report.type,
+ primary_identifier_fingerprint: primary_identifier&.fingerprint,
+ location_fingerprint: location_fingerprint,
+ project_id: report.project_id
+ }
+
+ if uuid_v5_name_components.values.any?(&:nil?)
+ Gitlab::AppLogger.warn(message: "One or more UUID name components are nil", components: uuid_v5_name_components)
+ return
+ end
+
+ ::Security::VulnerabilityUUID.generate(
+ report_type: uuid_v5_name_components[:report_type],
+ primary_identifier_fingerprint: uuid_v5_name_components[:primary_identifier_fingerprint],
+ location_fingerprint: uuid_v5_name_components[:location_fingerprint],
+ project_id: uuid_v5_name_components[:project_id]
+ )
+ end
+ end
+ end
+ end
+ end
+end
+
+Gitlab::Ci::Parsers::Security::Common.prepend_mod_with("Gitlab::Ci::Parsers::Security::Common")
diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb
new file mode 100644
index 00000000000..24613a441be
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Security
+ module Concerns
+ module DeprecatedSyntax
+ extend ActiveSupport::Concern
+
+ included do
+ extend ::Gitlab::Utils::Override
+
+ override :parse_report
+ end
+
+ def report_data
+ @report_data ||= begin
+ data = super
+
+ if data.is_a?(Array)
+ data = {
+ "version" => self.class::DEPRECATED_REPORT_VERSION,
+ "vulnerabilities" => data
+ }
+ end
+
+ data
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb
new file mode 100644
index 00000000000..e3c62614cd8
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/sast.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Security
+ class Sast < Common
+ include Security::Concerns::DeprecatedSyntax
+
+ DEPRECATED_REPORT_VERSION = "1.2"
+
+ private
+
+ def create_location(location_data)
+ ::Gitlab::Ci::Reports::Security::Locations::Sast.new(
+ file_path: location_data['file'],
+ start_line: location_data['start_line'],
+ end_line: location_data['end_line'],
+ class_name: location_data['class'],
+ method_name: location_data['method'])
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb
new file mode 100644
index 00000000000..c6d95c1d391
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/secret_detection.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Security
+ class SecretDetection < Common
+ include Security::Concerns::DeprecatedSyntax
+
+ DEPRECATED_REPORT_VERSION = "1.2"
+
+ private
+
+ def create_location(location_data)
+ ::Gitlab::Ci::Reports::Security::Locations::SecretDetection.new(
+ file_path: location_data['file'],
+ start_line: location_data['start_line'],
+ end_line: location_data['end_line'],
+ class_name: location_data['class'],
+ method_name: location_data['method']
+ )
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
new file mode 100644
index 00000000000..3d92886cba8
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Parsers
+ module Security
+ module Validators
+ class SchemaValidator
+ class Schema
+ def root_path
+ File.join(__dir__, 'schemas')
+ end
+
+ def initialize(report_type)
+ @report_type = report_type
+ end
+
+ delegate :validate, to: :schemer
+
+ private
+
+ attr_reader :report_type
+
+ def schemer
+ JSONSchemer.schema(pathname)
+ end
+
+ def pathname
+ Pathname.new(schema_path)
+ end
+
+ def schema_path
+ File.join(root_path, file_name)
+ end
+
+ def file_name
+ "#{report_type}.json"
+ end
+ end
+
+ def initialize(report_type, report_data)
+ @report_type = report_type
+ @report_data = report_data
+ end
+
+ def valid?
+ errors.empty?
+ end
+
+ def errors
+ @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+ end
+
+ private
+
+ attr_reader :report_type, :report_data
+
+ def schema
+ Schema.new(report_type)
+ end
+ end
+ end
+ end
+ end
+ end
+end
+
+Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema.prepend_mod_with("Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema")
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
new file mode 100644
index 00000000000..a7159be0190
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json
@@ -0,0 +1,706 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Report format for GitLab SAST",
+ "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "14.0.0"
+ },
+ "required": [
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "type": "object",
+ "description": "The vendor/maintainer of the scanner.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "sast"
+ ]
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "format": "uri"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability.",
+ "required": [
+ "category",
+ "cve",
+ "identifiers",
+ "location",
+ "scanner"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "category": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+ },
+ "name": {
+ "type": "string",
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "message": {
+ "type": "string",
+ "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "cve": {
+ "type": "string",
+ "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "confidence": {
+ "type": "string",
+ "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Ignore",
+ "Unknown",
+ "Experimental",
+ "Low",
+ "Medium",
+ "High",
+ "Confirmed"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "scanner": {
+ "description": "Describes the scanner used to find this vulnerability.",
+ "type": "object",
+ "required": [
+ "id",
+ "name"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "The scanner's ID, as a snake_case string."
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Human-readable name of the scanner."
+ }
+ }
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "format": "uri"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "format": "uri"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "location": {
+ "type": "object",
+ "description": "Identifies the vulnerability's location.",
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located."
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the code affected by the vulnerability."
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the code affected by the vulnerability."
+ },
+ "class": {
+ "type": "string",
+ "description": "Provides the name of the class where the vulnerability is located."
+ },
+ "method": {
+ "type": "string",
+ "description": "Provides the name of the method where the vulnerability is located."
+ }
+ }
+ },
+ "raw_source_code_extract": {
+ "type": "string",
+ "description": "Provides an unsanitized excerpt of the affected source code."
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "cve"
+ ],
+ "properties": {
+ "cve": {
+ "type": "string",
+ "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
new file mode 100644
index 00000000000..462e23a151c
--- /dev/null
+++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json
@@ -0,0 +1,729 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Report format for GitLab Secret Detection",
+ "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)",
+ "definitions": {
+ "detail_type": {
+ "oneOf": [
+ {
+ "$ref": "#/definitions/named_list"
+ },
+ {
+ "$ref": "#/definitions/list"
+ },
+ {
+ "$ref": "#/definitions/table"
+ },
+ {
+ "$ref": "#/definitions/text"
+ },
+ {
+ "$ref": "#/definitions/url"
+ },
+ {
+ "$ref": "#/definitions/code"
+ },
+ {
+ "$ref": "#/definitions/value"
+ },
+ {
+ "$ref": "#/definitions/diff"
+ },
+ {
+ "$ref": "#/definitions/markdown"
+ },
+ {
+ "$ref": "#/definitions/commit"
+ },
+ {
+ "$ref": "#/definitions/file_location"
+ },
+ {
+ "$ref": "#/definitions/module_location"
+ }
+ ]
+ },
+ "text_value": {
+ "type": "string"
+ },
+ "named_field": {
+ "type": "object",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "$ref": "#/definitions/text_value",
+ "minLength": 1
+ },
+ "description": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "named_list": {
+ "type": "object",
+ "description": "An object with named and typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "named-list"
+ },
+ "items": {
+ "type": "object",
+ "patternProperties": {
+ "^.*$": {
+ "allOf": [
+ {
+ "$ref": "#/definitions/named_field"
+ },
+ {
+ "$ref": "#/definitions/detail_type"
+ }
+ ]
+ }
+ }
+ }
+ }
+ },
+ "list": {
+ "type": "object",
+ "description": "A list of typed fields",
+ "required": [
+ "type",
+ "items"
+ ],
+ "properties": {
+ "type": {
+ "const": "list"
+ },
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ },
+ "table": {
+ "type": "object",
+ "description": "A table of typed fields",
+ "required": [
+ "type",
+ "rows"
+ ],
+ "properties": {
+ "type": {
+ "const": "table"
+ },
+ "header": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ },
+ "rows": {
+ "type": "array",
+ "items": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/detail_type"
+ }
+ }
+ }
+ }
+ },
+ "text": {
+ "type": "object",
+ "description": "Raw text",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "text"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value"
+ }
+ }
+ },
+ "url": {
+ "type": "object",
+ "description": "A single URL",
+ "required": [
+ "type",
+ "href"
+ ],
+ "properties": {
+ "type": {
+ "const": "url"
+ },
+ "text": {
+ "$ref": "#/definitions/text_value"
+ },
+ "href": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "http://mysite.com"
+ ]
+ }
+ }
+ },
+ "code": {
+ "type": "object",
+ "description": "A codeblock",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "code"
+ },
+ "value": {
+ "type": "string"
+ },
+ "lang": {
+ "type": "string",
+ "description": "A programming language"
+ }
+ }
+ },
+ "value": {
+ "type": "object",
+ "description": "A field that can store a range of types of value",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "value"
+ },
+ "value": {
+ "type": [
+ "number",
+ "string",
+ "boolean"
+ ]
+ }
+ }
+ },
+ "diff": {
+ "type": "object",
+ "description": "A diff",
+ "required": [
+ "type",
+ "before",
+ "after"
+ ],
+ "properties": {
+ "type": {
+ "const": "diff"
+ },
+ "before": {
+ "type": "string"
+ },
+ "after": {
+ "type": "string"
+ }
+ }
+ },
+ "markdown": {
+ "type": "object",
+ "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "markdown"
+ },
+ "value": {
+ "$ref": "#/definitions/text_value",
+ "examples": [
+ "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)"
+ ]
+ }
+ }
+ },
+ "commit": {
+ "type": "object",
+ "description": "A commit/tag/branch within the GitLab project",
+ "required": [
+ "type",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "const": "commit"
+ },
+ "value": {
+ "type": "string",
+ "description": "The commit SHA",
+ "minLength": 1
+ }
+ }
+ },
+ "file_location": {
+ "type": "object",
+ "description": "A location within a file in the project",
+ "required": [
+ "type",
+ "file_name",
+ "line_start"
+ ],
+ "properties": {
+ "type": {
+ "const": "file-location"
+ },
+ "file_name": {
+ "type": "string",
+ "minLength": 1
+ },
+ "line_start": {
+ "type": "integer"
+ },
+ "line_end": {
+ "type": "integer"
+ }
+ }
+ },
+ "module_location": {
+ "type": "object",
+ "description": "A location within a binary module of the form module+relative_offset",
+ "required": [
+ "type",
+ "module_name",
+ "offset"
+ ],
+ "properties": {
+ "type": {
+ "const": "module-location"
+ },
+ "module_name": {
+ "type": "string",
+ "minLength": 1,
+ "examples": [
+ "compiled_binary"
+ ]
+ },
+ "offset": {
+ "type": "integer",
+ "examples": [
+ 100
+ ]
+ }
+ }
+ }
+ },
+ "self": {
+ "version": "14.0.0"
+ },
+ "required": [
+ "version",
+ "vulnerabilities"
+ ],
+ "additionalProperties": true,
+ "properties": {
+ "scan": {
+ "type": "object",
+ "required": [
+ "end_time",
+ "scanner",
+ "start_time",
+ "status",
+ "type"
+ ],
+ "properties": {
+ "end_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "examples": [
+ "2020-01-28T03:26:02"
+ ]
+ },
+ "messages": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "description": "Communication intended for the initiator of a scan.",
+ "required": [
+ "level",
+ "value"
+ ],
+ "properties": {
+ "level": {
+ "type": "string",
+ "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.",
+ "enum": [
+ "info",
+ "warn",
+ "fatal"
+ ],
+ "examples": [
+ "info"
+ ]
+ },
+ "value": {
+ "type": "string",
+ "description": "The message to communicate.",
+ "minLength": 1,
+ "examples": [
+ "Permission denied, scanning aborted"
+ ]
+ }
+ }
+ }
+ },
+ "scanner": {
+ "type": "object",
+ "description": "Object defining the scanner used to perform the scan.",
+ "required": [
+ "id",
+ "name",
+ "version",
+ "vendor"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique id that identifies the scanner.",
+ "minLength": 1,
+ "examples": [
+ "my-sast-scanner"
+ ]
+ },
+ "name": {
+ "type": "string",
+ "description": "A human readable value that identifies the scanner, not required to be unique.",
+ "minLength": 1,
+ "examples": [
+ "My SAST Scanner"
+ ]
+ },
+ "url": {
+ "type": "string",
+ "description": "A link to more information about the scanner.",
+ "examples": [
+ "https://scanner.url"
+ ]
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the scanner.",
+ "minLength": 1,
+ "examples": [
+ "1.0.2"
+ ]
+ },
+ "vendor": {
+ "type": "object",
+ "description": "The vendor/maintainer of the scanner.",
+ "required": [
+ "name"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "The name of the vendor.",
+ "minLength": 1,
+ "examples": [
+ "GitLab"
+ ]
+ }
+ }
+ }
+ }
+ },
+ "start_time": {
+ "type": "string",
+ "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.",
+ "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$",
+ "examples": [
+ "2020-02-14T16:01:59"
+ ]
+ },
+ "status": {
+ "type": "string",
+ "description": "Result of the scan.",
+ "enum": [
+ "success",
+ "failure"
+ ]
+ },
+ "type": {
+ "type": "string",
+ "description": "Type of the scan.",
+ "enum": [
+ "secret_detection"
+ ]
+ }
+ }
+ },
+ "schema": {
+ "type": "string",
+ "description": "URI pointing to the validating security report schema.",
+ "format": "uri"
+ },
+ "version": {
+ "type": "string",
+ "description": "The version of the schema to which the JSON report conforms.",
+ "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$"
+ },
+ "vulnerabilities": {
+ "type": "array",
+ "description": "Array of vulnerability objects.",
+ "items": {
+ "type": "object",
+ "description": "Describes the vulnerability.",
+ "required": [
+ "category",
+ "cve",
+ "identifiers",
+ "location",
+ "scanner"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.",
+ "examples": [
+ "642735a5-1425-428d-8d4e-3c854885a3c9"
+ ]
+ },
+ "category": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)."
+ },
+ "name": {
+ "type": "string",
+ "description": "The name of the vulnerability. This must not include the finding's specific information."
+ },
+ "message": {
+ "type": "string",
+ "description": "A short text section that describes the vulnerability. This may include the finding's specific information."
+ },
+ "description": {
+ "type": "string",
+ "description": "A long text section describing the vulnerability more fully."
+ },
+ "cve": {
+ "type": "string",
+ "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ },
+ "severity": {
+ "type": "string",
+ "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Info",
+ "Unknown",
+ "Low",
+ "Medium",
+ "High",
+ "Critical"
+ ]
+ },
+ "confidence": {
+ "type": "string",
+ "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.",
+ "enum": [
+ "Ignore",
+ "Unknown",
+ "Experimental",
+ "Low",
+ "Medium",
+ "High",
+ "Confirmed"
+ ]
+ },
+ "solution": {
+ "type": "string",
+ "description": "Explanation of how to fix the vulnerability."
+ },
+ "scanner": {
+ "description": "Describes the scanner used to find this vulnerability.",
+ "type": "object",
+ "required": [
+ "id",
+ "name"
+ ],
+ "properties": {
+ "id": {
+ "type": "string",
+ "minLength": 1,
+ "description": "The scanner's ID, as a snake_case string."
+ },
+ "name": {
+ "type": "string",
+ "minLength": 1,
+ "description": "Human-readable name of the scanner."
+ }
+ }
+ },
+ "identifiers": {
+ "type": "array",
+ "minItems": 1,
+ "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.",
+ "items": {
+ "type": "object",
+ "required": [
+ "type",
+ "name",
+ "value"
+ ],
+ "properties": {
+ "type": {
+ "type": "string",
+ "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).",
+ "minLength": 1
+ },
+ "name": {
+ "type": "string",
+ "description": "Human-readable name of the identifier.",
+ "minLength": 1
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the identifier's documentation.",
+ "format": "uri"
+ },
+ "value": {
+ "type": "string",
+ "description": "Value of the identifier, for matching purpose.",
+ "minLength": 1
+ }
+ }
+ }
+ },
+ "links": {
+ "type": "array",
+ "description": "An array of references to external documentation or articles that describe the vulnerability.",
+ "items": {
+ "type": "object",
+ "required": [
+ "url"
+ ],
+ "properties": {
+ "name": {
+ "type": "string",
+ "description": "Name of the vulnerability details link."
+ },
+ "url": {
+ "type": "string",
+ "description": "URL of the vulnerability details document.",
+ "format": "uri"
+ }
+ }
+ }
+ },
+ "details": {
+ "$ref": "#/definitions/named_list/properties/items"
+ },
+ "location": {
+ "required": [
+ "commit"
+ ],
+ "properties": {
+ "file": {
+ "type": "string",
+ "description": "Path to the file where the vulnerability is located"
+ },
+ "commit": {
+ "type": "object",
+ "description": "Represents the commit in which the vulnerability was detected",
+ "required": [
+ "sha"
+ ],
+ "properties": {
+ "author": {
+ "type": "string"
+ },
+ "date": {
+ "type": "string"
+ },
+ "message": {
+ "type": "string"
+ },
+ "sha": {
+ "type": "string",
+ "minLength": 1
+ }
+ }
+ },
+ "start_line": {
+ "type": "number",
+ "description": "The first line of the code affected by the vulnerability"
+ },
+ "end_line": {
+ "type": "number",
+ "description": "The last line of the code affected by the vulnerability"
+ },
+ "class": {
+ "type": "string",
+ "description": "Provides the name of the class where the vulnerability is located"
+ },
+ "method": {
+ "type": "string",
+ "description": "Provides the name of the method where the vulnerability is located"
+ }
+ }
+ },
+ "raw_source_code_extract": {
+ "type": "string",
+ "description": "Provides an unsanitized excerpt of the affected source code."
+ }
+ }
+ }
+ },
+ "remediations": {
+ "type": "array",
+ "description": "An array of objects containing information on available remediations, along with patch diffs to apply.",
+ "items": {
+ "type": "object",
+ "required": [
+ "fixes",
+ "summary",
+ "diff"
+ ],
+ "properties": {
+ "fixes": {
+ "type": "array",
+ "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.",
+ "items": {
+ "type": "object",
+ "required": [
+ "cve"
+ ],
+ "properties": {
+ "cve": {
+ "type": "string",
+ "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/."
+ }
+ }
+ }
+ },
+ "summary": {
+ "type": "string",
+ "minLength": 1,
+ "description": "An overview of how the vulnerabilities were fixed."
+ },
+ "diff": {
+ "type": "string",
+ "minLength": 1,
+ "description": "A base64-encoded remediation code diff, compatible with git apply."
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb
index 7564d0c3ed5..626eba97817 100644
--- a/lib/gitlab/ci/pipeline/chain/command.rb
+++ b/lib/gitlab/ci/pipeline/chain/command.rb
@@ -97,15 +97,16 @@ module Gitlab
.observe({ source: pipeline.source.to_s }, pipeline.total_size)
end
+ def observe_jobs_count_in_alive_pipelines
+ metrics.active_jobs_histogram
+ .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines)
+ end
+
def increment_pipeline_failure_reason_counter(reason)
metrics.pipeline_failure_reason_counter
.increment(reason: (reason || :unknown_failure).to_s)
end
- def dangling_build?
- %i[ondemand_dast_scan webide].include?(source)
- end
-
private
# Verifies that origin_ref is a fully qualified tag reference (refs/tags/<tag-name>)
diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb
index 49ec1250a5f..5251dd3d40a 100644
--- a/lib/gitlab/ci/pipeline/chain/config/process.rb
+++ b/lib/gitlab/ci/pipeline/chain/config/process.rb
@@ -14,7 +14,7 @@ module Gitlab
result = ::Gitlab::Ci::YamlProcessor.new(
@command.config_content, {
project: project,
- ref: @pipeline.ref,
+ source_ref_path: @pipeline.source_ref_path,
sha: @pipeline.sha,
source: @pipeline.source,
user: current_user,
diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb
index dc648568129..bbfc6759b35 100644
--- a/lib/gitlab/ci/pipeline/chain/sequence.rb
+++ b/lib/gitlab/ci/pipeline/chain/sequence.rb
@@ -22,6 +22,7 @@ module Gitlab
@command.observe_creation_duration(Time.now - @start)
@command.observe_pipeline_size(@pipeline)
+ @command.observe_jobs_count_in_alive_pipelines
@pipeline
end
diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb
index e4e4f4f484a..76dfb4cbd87 100644
--- a/lib/gitlab/ci/pipeline/chain/skip.rb
+++ b/lib/gitlab/ci/pipeline/chain/skip.rb
@@ -22,16 +22,16 @@ module Gitlab
end
end
- def skipped?
- !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?)
- end
-
def break?
skipped?
end
private
+ def skipped?
+ !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?)
+ end
+
def commit_message_skips_ci?
return false unless @pipeline.git_commit_message
diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
index 514241e8ae2..c7106f3ec39 100644
--- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
+++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb
@@ -11,7 +11,7 @@ module Gitlab
PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze
def initialize(regexp)
- super(regexp.gsub(/\\\//, '/'))
+ super(regexp.gsub(%r{\\/}, '/'))
unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value)
raise Lexer::SyntaxError, 'Invalid regular expression!'
diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb
index 84b88374a7f..10de77afe74 100644
--- a/lib/gitlab/ci/pipeline/metrics.rb
+++ b/lib/gitlab/ci/pipeline/metrics.rb
@@ -24,7 +24,16 @@ module Gitlab
name = :gitlab_ci_pipeline_size_builds
comment = 'Pipeline size'
labels = { source: nil }
- buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000]
+ buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 3000]
+
+ ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+ end
+
+ def self.active_jobs_histogram
+ name = :gitlab_ci_active_jobs
+ comment = 'Total amount of active jobs'
+ labels = { plan: nil }
+ buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000]
::Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb
index 54d92745992..c393fed26de 100644
--- a/lib/gitlab/ci/pipeline/seed/build.rb
+++ b/lib/gitlab/ci/pipeline/seed/build.rb
@@ -39,7 +39,7 @@ module Gitlab
@cache = Gitlab::Ci::Build::Cache
.new(attributes.delete(:cache), @pipeline)
- recalculate_yaml_variables!
+ calculate_yaml_variables!
end
def name
@@ -232,7 +232,7 @@ module Gitlab
{ options: { allow_failure_criteria: nil } }
end
- def recalculate_yaml_variables!
+ def calculate_yaml_variables!
@seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables(
from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance
)
diff --git a/lib/gitlab/ci/reports/security/aggregated_report.rb b/lib/gitlab/ci/reports/security/aggregated_report.rb
new file mode 100644
index 00000000000..a8bb2196043
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/aggregated_report.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+# Used to represent combined Security Reports. This is typically done for vulnerability deduplication purposes.
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class AggregatedReport
+ attr_reader :findings
+
+ def initialize(reports, findings)
+ @reports = reports
+ @findings = findings
+ end
+
+ def created_at
+ @reports.map(&:created_at).compact.min
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb
new file mode 100644
index 00000000000..dc1c51b3ed0
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding.rb
@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class Finding
+ include ::VulnerabilityFindingHelpers
+
+ attr_reader :compare_key
+ attr_reader :confidence
+ attr_reader :identifiers
+ attr_reader :links
+ attr_reader :location
+ attr_reader :metadata_version
+ attr_reader :name
+ attr_reader :old_location
+ attr_reader :project_fingerprint
+ attr_reader :raw_metadata
+ attr_reader :report_type
+ attr_reader :scanner
+ attr_reader :scan
+ attr_reader :severity
+ attr_accessor :uuid
+ attr_accessor :overridden_uuid
+ attr_reader :remediations
+ attr_reader :details
+ attr_reader :signatures
+ attr_reader :project_id
+
+ delegate :file_path, :start_line, :end_line, to: :location
+
+ def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists
+ @compare_key = compare_key
+ @confidence = confidence
+ @identifiers = identifiers
+ @links = links
+ @location = location
+ @metadata_version = metadata_version
+ @name = name
+ @raw_metadata = raw_metadata
+ @report_type = report_type
+ @scanner = scanner
+ @scan = scan
+ @severity = severity
+ @uuid = uuid
+ @remediations = remediations
+ @details = details
+ @signatures = signatures
+ @project_id = project_id
+ @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
+
+ @project_fingerprint = generate_project_fingerprint
+ end
+
+ def to_hash
+ %i[
+ compare_key
+ confidence
+ identifiers
+ links
+ location
+ metadata_version
+ name
+ project_fingerprint
+ raw_metadata
+ report_type
+ scanner
+ scan
+ severity
+ uuid
+ details
+ signatures
+ ].each_with_object({}) do |key, hash|
+ hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+
+ def primary_identifier
+ identifiers.first
+ end
+
+ def update_location(new_location)
+ @old_location = location
+ @location = new_location
+ end
+
+ def unsafe?(severity_levels)
+ severity.in?(severity_levels)
+ end
+
+ def eql?(other)
+ return false unless report_type == other.report_type && primary_identifier_fingerprint == other.primary_identifier_fingerprint
+
+ if @vulnerability_finding_signatures_enabled
+ matches_signatures(other.signatures, other.uuid)
+ else
+ location.fingerprint == other.location.fingerprint
+ end
+ end
+
+ def hash
+ if @vulnerability_finding_signatures_enabled && !signatures.empty?
+ highest_signature = signatures.max_by(&:priority)
+ report_type.hash ^ highest_signature.signature_hex.hash ^ primary_identifier_fingerprint.hash
+ else
+ report_type.hash ^ location.fingerprint.hash ^ primary_identifier_fingerprint.hash
+ end
+ end
+
+ def valid?
+ scanner.present? && primary_identifier.present? && location.present? && uuid.present?
+ end
+
+ def keys
+ @keys ||= identifiers.reject(&:type_identifier?).map do |identifier|
+ FindingKey.new(location_fingerprint: location&.fingerprint, identifier_fingerprint: identifier.fingerprint)
+ end
+ end
+
+ def primary_identifier_fingerprint
+ primary_identifier&.fingerprint
+ end
+
+ def <=>(other)
+ if severity == other.severity
+ compare_key <=> other.compare_key
+ else
+ ::Enums::Vulnerability.severity_levels[other.severity] <=>
+ ::Enums::Vulnerability.severity_levels[severity]
+ end
+ end
+
+ def scanner_order_to(other)
+ return 1 unless scanner
+ return -1 unless other&.scanner
+
+ scanner <=> other.scanner
+ end
+
+ private
+
+ def generate_project_fingerprint
+ Digest::SHA1.hexdigest(compare_key)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb
new file mode 100644
index 00000000000..0acd923a60f
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding_key.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class FindingKey
+ def initialize(location_fingerprint:, identifier_fingerprint:)
+ @location_fingerprint = location_fingerprint
+ @identifier_fingerprint = identifier_fingerprint
+ end
+
+ def ==(other)
+ has_fingerprints? && other.has_fingerprints? &&
+ location_fingerprint == other.location_fingerprint &&
+ identifier_fingerprint == other.identifier_fingerprint
+ end
+
+ def hash
+ location_fingerprint.hash ^ identifier_fingerprint.hash
+ end
+
+ alias_method :eql?, :==
+
+ protected
+
+ attr_reader :location_fingerprint, :identifier_fingerprint
+
+ def has_fingerprints?
+ location_fingerprint.present? && identifier_fingerprint.present?
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/finding_signature.rb b/lib/gitlab/ci/reports/security/finding_signature.rb
new file mode 100644
index 00000000000..d1d7ef5c377
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/finding_signature.rb
@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class FindingSignature
+ include VulnerabilityFindingSignatureHelpers
+
+ attr_accessor :algorithm_type, :signature_value
+
+ def initialize(params = {})
+ @algorithm_type = params.dig(:algorithm_type)
+ @signature_value = params.dig(:signature_value)
+ end
+
+ def signature_sha
+ Digest::SHA1.digest(signature_value)
+ end
+
+ def signature_hex
+ signature_sha.unpack1("H*")
+ end
+
+ def to_hash
+ {
+ algorithm_type: algorithm_type,
+ signature_sha: signature_sha
+ }
+ end
+
+ def valid?
+ algorithm_types.key?(algorithm_type)
+ end
+
+ def eql?(other)
+ other.algorithm_type == algorithm_type &&
+ other.signature_sha == signature_sha
+ end
+
+ alias_method :==, :eql?
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/base.rb b/lib/gitlab/ci/reports/security/locations/base.rb
new file mode 100644
index 00000000000..9ad1d81287f
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/base.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ module Locations
+ class Base
+ include ::Gitlab::Utils::StrongMemoize
+
+ def ==(other)
+ other.fingerprint == fingerprint
+ end
+
+ def fingerprint
+ strong_memoize(:fingerprint) do
+ Digest::SHA1.hexdigest(fingerprint_data)
+ end
+ end
+
+ def as_json(options = nil)
+ fingerprint # side-effect call to initialize the ivar for serialization
+
+ super
+ end
+
+ def fingerprint_path
+ fingerprint_data
+ end
+
+ private
+
+ def fingerprint_data
+ raise NotImplementedError
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/sast.rb b/lib/gitlab/ci/reports/security/locations/sast.rb
new file mode 100644
index 00000000000..23ffa91e720
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/sast.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ module Locations
+ class Sast < Base
+ include Security::Concerns::FingerprintPathFromFile
+
+ attr_reader :class_name
+ attr_reader :end_line
+ attr_reader :file_path
+ attr_reader :method_name
+ attr_reader :start_line
+
+ def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil)
+ @class_name = class_name
+ @end_line = end_line
+ @file_path = file_path
+ @method_name = method_name
+ @start_line = start_line
+ end
+
+ def fingerprint_data
+ "#{file_path}:#{start_line}:#{end_line}"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/locations/secret_detection.rb b/lib/gitlab/ci/reports/security/locations/secret_detection.rb
new file mode 100644
index 00000000000..0fd5cc5af11
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/locations/secret_detection.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ module Locations
+ class SecretDetection < Base
+ include Security::Concerns::FingerprintPathFromFile
+
+ attr_reader :class_name
+ attr_reader :end_line
+ attr_reader :file_path
+ attr_reader :method_name
+ attr_reader :start_line
+
+ def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil)
+ @class_name = class_name
+ @end_line = end_line
+ @file_path = file_path
+ @method_name = method_name
+ @start_line = start_line
+ end
+
+ def fingerprint_data
+ "#{file_path}:#{start_line}:#{end_line}"
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb
new file mode 100644
index 00000000000..1ba2d909d99
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/report.rb
@@ -0,0 +1,76 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class Report
+ attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers
+ attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version
+
+ delegate :project_id, to: :pipeline
+
+ def initialize(type, pipeline, created_at)
+ @type = type
+ @pipeline = pipeline
+ @created_at = created_at
+ @findings = []
+ @scanners = {}
+ @identifiers = {}
+ @scanned_resources = []
+ @errors = []
+ end
+
+ def commit_sha
+ pipeline.sha
+ end
+
+ def add_error(type, message = 'An unexpected error happened!')
+ errors << { type: type, message: message }
+ end
+
+ def errored?
+ errors.present?
+ end
+
+ def add_scanner(scanner)
+ scanners[scanner.key] ||= scanner
+ end
+
+ def add_identifier(identifier)
+ identifiers[identifier.key] ||= identifier
+ end
+
+ def add_finding(finding)
+ findings << finding
+ end
+
+ def clone_as_blank
+ Report.new(type, pipeline, created_at)
+ end
+
+ def replace_with!(other)
+ instance_variables.each do |ivar|
+ instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend
+ end
+ end
+
+ def merge!(other)
+ replace_with!(::Security::MergeReportsService.new(self, other).execute)
+ end
+
+ def primary_scanner
+ scanners.first&.second
+ end
+
+ def primary_scanner_order_to(other)
+ return 1 unless primary_scanner
+ return -1 unless other.primary_scanner
+
+ primary_scanner <=> other.primary_scanner
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb
new file mode 100644
index 00000000000..b7a5e36b108
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/reports.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class Reports
+ attr_reader :reports, :pipeline
+
+ delegate :each, :empty?, to: :reports
+
+ def initialize(pipeline)
+ @reports = {}
+ @pipeline = pipeline
+ end
+
+ def get_report(report_type, report_artifact)
+ reports[report_type] ||= Report.new(report_type, pipeline, report_artifact.created_at)
+ end
+
+ def findings
+ reports.values.flat_map(&:findings)
+ end
+
+ def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels)
+ unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed
+ end
+
+ private
+
+ def findings_diff(target_reports)
+ findings - target_reports&.findings.to_a
+ end
+
+ def unsafe_findings_count(target_reports, severity_levels)
+ findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)}
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
new file mode 100644
index 00000000000..6cb2e0ddb33
--- /dev/null
+++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb
@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Ci
+ module Reports
+ module Security
+ class VulnerabilityReportsComparer
+ include Gitlab::Utils::StrongMemoize
+
+ attr_reader :base_report, :head_report
+
+ ACCEPTABLE_REPORT_AGE = 1.week
+
+ def initialize(project, base_report, head_report)
+ @base_report = base_report
+ @head_report = head_report
+
+ @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures)
+
+ if @signatures_enabled
+ @added_findings = []
+ @fixed_findings = []
+ calculate_changes
+ end
+ end
+
+ def base_report_created_at
+ @base_report.created_at
+ end
+
+ def head_report_created_at
+ @head_report.created_at
+ end
+
+ def base_report_out_of_date
+ return false unless @base_report.created_at
+
+ ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at
+ end
+
+ def added
+ strong_memoize(:added) do
+ if @signatures_enabled
+ @added_findings
+ else
+ head_report.findings - base_report.findings
+ end
+ end
+ end
+
+ def fixed
+ strong_memoize(:fixed) do
+ if @signatures_enabled
+ @fixed_findings
+ else
+ base_report.findings - head_report.findings
+ end
+ end
+ end
+
+ private
+
+ def calculate_changes
+ # This is a deconstructed version of the eql? method on
+ # Ci::Reports::Security::Finding. It:
+ #
+ # * precomputes for the head_findings (using FindingMatcher):
+ # * sets of signature shas grouped by priority
+ # * mappings of signature shas to the head finding object
+ #
+ # These are then used when iterating the base findings to perform
+ # fast(er) prioritized, signature-based comparisons between each base finding
+ # and the head findings.
+ #
+ # Both the head_findings and base_findings arrays are iterated once
+
+ base_findings = base_report.findings
+ head_findings = head_report.findings
+
+ matcher = FindingMatcher.new(head_findings)
+
+ base_findings.each do |base_finding|
+ matched_head_finding = matcher.find_and_remove_match!(base_finding)
+
+ @fixed_findings << base_finding if matched_head_finding.nil?
+ end
+
+ @added_findings = matcher.unmatched_head_findings.values
+ end
+ end
+
+ class FindingMatcher
+ attr_reader :unmatched_head_findings, :head_findings
+
+ include Gitlab::Utils::StrongMemoize
+
+ def initialize(head_findings)
+ @head_findings = head_findings
+ @unmatched_head_findings = @head_findings.index_by(&:object_id)
+ end
+
+ def find_and_remove_match!(base_finding)
+ matched_head_finding = find_matched_head_finding_for(base_finding)
+
+ # no signatures matched, so check the normal uuids of the base and head findings
+ # for a match
+ matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil?
+
+ @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil?
+
+ matched_head_finding
+ end
+
+ private
+
+ def find_matched_head_finding_for(base_finding)
+ base_signature = sorted_signatures_for(base_finding).find do |signature|
+ # at this point a head_finding exists that has a signature with a
+ # matching priority, and a matching sha --> lookup the actual finding
+ # object from head_signatures_shas
+ head_signatures_shas[signature.signature_sha].eql?(base_finding)
+ end
+
+ base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil
+ end
+
+ def sorted_signatures_for(base_finding)
+ base_finding.signatures.select { |signature| head_finding_signature?(signature) }
+ .sort_by { |sig| -sig.priority }
+ end
+
+ def head_finding_signature?(signature)
+ head_signatures_priorities[signature.priority].include?(signature.signature_sha)
+ end
+
+ def head_signatures_priorities
+ strong_memoize(:head_signatures_priorities) do
+ signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new }
+
+ head_findings.each_with_object(signatures_priorities) do |head_finding, memo|
+ head_finding.signatures.each do |signature|
+ memo[signature.priority].add(signature.signature_sha)
+ end
+ end
+ end
+ end
+
+ def head_signatures_shas
+ strong_memoize(:head_signatures_shas) do
+ head_findings.each_with_object({}) do |head_finding, memo|
+ head_finding.signatures.each do |signature|
+ memo[signature.signature_sha] = head_finding
+ end
+ # for the final uuid check when no signatures have matched
+ memo[head_finding.uuid] = head_finding
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
index ebb0b5948f1..71f38ededd9 100644
--- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml
@@ -5,7 +5,7 @@
# This template is on early stage of development.
# Use it with caution. For usage instruction please read
-# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md
+# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v3.0.0/README.md
include:
# workflow rules to prevent duplicate detached pipelines
diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
index 1910913f2bd..f39a84bceec 100644
--- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
@@ -3,7 +3,7 @@
# This specific template is located at:
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml
-# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options
+# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options
# you can delete this line if you're not using Docker
image: busybox:latest
diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
index d2d3b3ed61e..f147ad9332d 100644
--- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
POSTGRES_DB: database_name
# This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
cache:
paths:
- ~/.cache/pip/
diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
index 38036c1f964..21a599fc78d 100644
--- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
@@ -10,7 +10,7 @@
# A pipeline is composed of independent jobs that run scripts, grouped into stages.
# Stages run in sequential order, but jobs within stages run in parallel.
#
-# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
stages: # List of stages for jobs, and their order of execution
- build
diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
index 48e877684f6..43ecc4b96d5 100644
--- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml
@@ -27,7 +27,7 @@ code_quality:
}
- docker pull --quiet "$CODE_QUALITY_IMAGE"
- |
- docker run \
+ docker run --rm \
$(propagate_env_vars \
SOURCE_CODE \
TIMEOUT_SECONDS \
diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
index 00fcfa64a18..208951fa1a1 100644
--- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.dast-auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
dast_environment_deploy:
extends: .dast-auto-deploy
diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
index 530ab1d0f99..5c466f0984c 100644
--- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml
@@ -1,5 +1,5 @@
.auto-deploy:
- image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0"
+ image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0"
dependencies: []
review:
diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
index 80125a9bc01..917a28bb1ee 100644
--- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml
@@ -252,6 +252,7 @@ semgrep-sast:
- '**/*.jsx'
- '**/*.ts'
- '**/*.tsx'
+ - '**/*.c'
sobelow-sast:
extends: .sast-analyzer
diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
index d0595491400..18f0f20203d 100644
--- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml
@@ -27,8 +27,8 @@ secret_detection:
when: never
- if: $CI_COMMIT_BRANCH
script:
- - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
- - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi
+ - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi
+ - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi
- git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME
- git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt
- export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt
diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
index 43e4ac02d41..ff7bac15017 100644
--- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
MYSQL_ROOT_PASSWORD: secret
# This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
cache:
paths:
- vendor/
diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
index e48801b7970..16bc0026aa8 100644
--- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml
@@ -16,7 +16,7 @@ services:
- postgres:latest
# This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
cache:
paths:
- node_modules/
diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
index d3726fe34c5..9da50439be8 100644
--- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml
@@ -6,7 +6,7 @@
image: node:latest
# This folder is cached between builds
-# http://docs.gitlab.com/ee/ci/yaml/README.html#cache
+# https://docs.gitlab.com/ee/ci/yaml/index.html#cache
cache:
paths:
- node_modules/
diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
index 490fc779e17..0c8b98dc1cf 100644
--- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml
@@ -29,7 +29,8 @@ before_script:
- ruby -v # Print out ruby version for debugging
# Uncomment next line if your rails app needs a JS runtime:
# - apt-get update -q && apt-get install nodejs -yqq
- - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby
+ - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby
+ - bundle install -j $(nproc)
# Optional - Delete if not using `rubocop`
rubocop:
diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
index f4f066cc7c2..ed4876c2bcc 100644
--- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml
@@ -8,7 +8,7 @@
# - A `test` stage to be present in the pipeline.
# - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities.
#
-# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html).
+# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html).
# List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables
variables:
diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
new file mode 100644
index 00000000000..d27a08db181
--- /dev/null
+++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
@@ -0,0 +1,23 @@
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml
+
+stages:
+ - build
+ - test
+ - deploy
+ - dast
+
+variables:
+ DAST_RUNNER_VALIDATION_VERSION: 1
+
+validation:
+ stage: dast
+ image:
+ name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION"
+ variables:
+ GIT_STRATEGY: none
+ allow_failure: false
+ script:
+ - ~/validate.sh
diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
index e30777d8401..86b7d57d3cb 100644
--- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml
@@ -18,7 +18,7 @@ variables:
bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep,
bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python,
license-finder,
- dast, api-fuzzing
+ dast, dast-runner-validation, api-fuzzing
SECURE_BINARIES_DOWNLOAD_IMAGES: "true"
SECURE_BINARIES_PUSH_IMAGES: "true"
@@ -230,6 +230,16 @@ dast:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bdast\b/
+dast-runner-validation:
+ extends: .download_images
+ variables:
+ SECURE_BINARIES_ANALYZER_VERSION: "1"
+ SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"
+ only:
+ variables:
+ - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
+ $SECURE_BINARIES_ANALYZERS =~ /\bdast-runner-validation\b/
+
api-fuzzing:
extends: .download_images
variables:
diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
index 272b980b4b2..1a857ef3eb3 100644
--- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
@@ -4,7 +4,7 @@
# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml
include:
- - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+ - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
stages:
- init
diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
index d34a847f2d5..a9f6fd88d0b 100644
--- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
@@ -14,15 +14,22 @@ stages:
- cleanup
init:
- extends: .init
+ extends: .terraform:init
validate:
- extends: .validate
+ extends: .terraform:validate
build:
- extends: .build
+ extends: .terraform:build
deploy:
- extends: .deploy
+ extends: .terraform:deploy
dependencies:
- build
+ environment:
+ name: $TF_STATE_NAME
+
+cleanup:
+ extends: .terraform:destroy
+ dependencies:
+ - deploy
diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
new file mode 100644
index 00000000000..39c3374e534
--- /dev/null
+++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml
@@ -0,0 +1,64 @@
+# Terraform/Base.latest
+#
+# The purpose of this template is to provide flexibility to the user so
+# they are able to only include the jobs that they find interesting.
+#
+# Therefore, this template is not supposed to run any jobs. The idea is to only
+# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs
+#
+# There is a more opinionated template which we suggest the users to abide,
+# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml
+
+image:
+ name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3
+
+variables:
+ TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project
+ TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend
+
+cache:
+ key: "${TF_ROOT}"
+ paths:
+ - ${TF_ROOT}/.terraform/
+ - ${TF_ROOT}/.terraform.lock.hcl
+
+.init: &init
+ stage: init
+ script:
+ - cd ${TF_ROOT}
+ - gitlab-terraform init
+
+.validate: &validate
+ stage: validate
+ script:
+ - cd ${TF_ROOT}
+ - gitlab-terraform validate
+
+.build: &build
+ stage: build
+ script:
+ - cd ${TF_ROOT}
+ - gitlab-terraform plan
+ - gitlab-terraform plan-json
+ artifacts:
+ paths:
+ - ${TF_ROOT}/plan.cache
+ reports:
+ terraform: ${TF_ROOT}/plan.json
+
+.deploy: &deploy
+ stage: deploy
+ script:
+ - cd ${TF_ROOT}
+ - gitlab-terraform apply
+ when: manual
+ only:
+ variables:
+ - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+
+.destroy: &destroy
+ stage: cleanup
+ script:
+ - cd ${TF_ROOT}
+ - gitlab-terraform destroy
+ when: manual
diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
index 200388a274c..c30860ad174 100644
--- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
+++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml
@@ -13,7 +13,8 @@ image:
name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest
variables:
- TF_ROOT: ${CI_PROJECT_DIR}
+ TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project
+ TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend
cache:
key: "${TF_ROOT}"
@@ -21,43 +22,46 @@ cache:
- ${TF_ROOT}/.terraform/
- ${TF_ROOT}/.terraform.lock.hcl
-.init: &init
+.terraform:init: &terraform_init
stage: init
script:
- cd ${TF_ROOT}
- gitlab-terraform init
-.validate: &validate
+.terraform:validate: &terraform_validate
stage: validate
script:
- cd ${TF_ROOT}
- gitlab-terraform validate
-.build: &build
+.terraform:build: &terraform_build
stage: build
script:
- cd ${TF_ROOT}
- gitlab-terraform plan
- gitlab-terraform plan-json
+ resource_group: ${TF_STATE_NAME}
artifacts:
paths:
- ${TF_ROOT}/plan.cache
reports:
terraform: ${TF_ROOT}/plan.json
-.deploy: &deploy
+.terraform:deploy: &terraform_deploy
stage: deploy
script:
- cd ${TF_ROOT}
- gitlab-terraform apply
+ resource_group: ${TF_STATE_NAME}
when: manual
only:
variables:
- $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-.destroy: &destroy
+.terraform:destroy: &terraform_destroy
stage: cleanup
script:
- cd ${TF_ROOT}
- gitlab-terraform destroy
+ resource_group: ${TF_STATE_NAME}
when: manual
diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb
index 0140218d9bc..8ab9573dd20 100644
--- a/lib/gitlab/ci/yaml_processor/dag.rb
+++ b/lib/gitlab/ci/yaml_processor/dag.rb
@@ -23,7 +23,7 @@ module Gitlab
new(nodes).tsort
rescue TSort::Cyclic
- raise ValidationError, 'The pipeline has circular dependencies.'
+ raise ValidationError, 'The pipeline has circular dependencies'
rescue MissingNodeError
end
diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb
index dd5107bad9a..a97c7050fbb 100644
--- a/lib/gitlab/ci/yaml_processor/result.rb
+++ b/lib/gitlab/ci/yaml_processor/result.rb
@@ -69,7 +69,7 @@ module Gitlab
when: job[:when] || 'on_success',
environment: job[:environment_name],
coverage_regex: job[:coverage],
- yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581
+ # yaml_variables is calculated with using job_variables in Seed::Build
job_variables: transform_to_yaml_variables(job[:job_variables]),
root_variables_inheritance: job[:root_variables_inheritance],
needs_attributes: job.dig(:needs, :job),