diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 12:08:42 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2021-08-19 12:08:42 +0300 |
commit | b76ae638462ab0f673e5915986070518dd3f9ad3 (patch) | |
tree | bdab0533383b52873be0ec0eb4d3c66598ff8b91 /lib/gitlab/ci | |
parent | 434373eabe7b4be9593d18a585fb763f1e5f1a6f (diff) |
Add latest changes from gitlab-org/gitlab@14-2-stable-eev14.2.0-rc42
Diffstat (limited to 'lib/gitlab/ci')
65 files changed, 2835 insertions, 110 deletions
diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 97988d8aa13..ef936581c10 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -33,6 +33,8 @@ module Gitlab Result = Struct.new(:html, :state, :append, :truncated, :offset, :size, :total, keyword_init: true) # rubocop:disable Lint/StructNewOverride class Converter + include EncodingHelper + def on_0(_) reset end @@ -256,6 +258,7 @@ module Gitlab start_offset = @offset stream.each_line do |line| + line = encode_utf8_no_detect(line) s = StringScanner.new(line) until s.eos? diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 8f2d47e7ccc..e48080993ab 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -9,6 +9,8 @@ module Gitlab # Line::Segment is a portion of a line that has its own style # and text. Multiple segments make the line content. class Segment + include EncodingHelper + attr_accessor :text, :style def initialize(style:) @@ -21,11 +23,12 @@ module Gitlab end def to_h - # Without force encoding to UTF-8 we could get an error - # when serializing the Hash to JSON. + # Without forcing the encoding to UTF-8 and then replacing + # invalid UTF-8 sequences we can get an error when serializing + # the Hash to JSON. # Encoding::UndefinedConversionError: # "\xE2" from ASCII-8BIT to UTF-8 - { text: text.force_encoding('UTF-8') }.tap do |result| + { text: encode_utf8_no_detect(text) }.tap do |result| result[:style] = style.to_s if style.set? end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 9c6428d701c..aceaf012f7e 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -17,13 +17,13 @@ module Gitlab Config::Yaml::Tags::TagError ].freeze - attr_reader :root, :context, :ref, :source + attr_reader :root, :context, :source_ref_path, :source - def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, ref: nil, source: nil) - @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline) + def initialize(config, project: nil, sha: nil, user: nil, parent_pipeline: nil, source_ref_path: nil, source: nil) + @context = build_context(project: project, sha: sha, user: user, parent_pipeline: parent_pipeline, ref: source_ref_path) @context.set_deadline(TIMEOUT_SECONDS) - @ref = ref + @source_ref_path = source_ref_path @source = source @config = expand_config(config) @@ -108,13 +108,37 @@ module Gitlab end end - def build_context(project:, sha:, user:, parent_pipeline:) + def build_context(project:, sha:, user:, parent_pipeline:, ref:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, - variables: project&.predefined_variables&.to_runner_variables) + variables: build_variables(project: project, ref: ref)) + end + + def build_variables(project:, ref:) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless project + + # The order of the following lines is important as priority of CI variables is + # defined globally within GitLab. + # + # See more detail in the docs: https://docs.gitlab.com/ee/ci/variables/#cicd-variable-precedence + variables.concat(project.predefined_variables) + variables.concat(pipeline_predefined_variables(ref: ref)) + variables.concat(project.ci_instance_variables_for(ref: ref)) + variables.concat(project.group.ci_variables_for(ref, project)) if project.group + variables.concat(project.ci_variables_for(ref: ref)) + end + end + + # https://gitlab.com/gitlab-org/gitlab/-/issues/337633 aims to add all predefined variables + # to this list, but only CI_COMMIT_REF_NAME is available right now to support compliance pipelines. + def pipeline_predefined_variables(ref:) + Gitlab::Ci::Variables::Collection.new.tap do |v| + v.append(key: 'CI_COMMIT_REF_NAME', value: ref) + end end def track_and_raise_for_dev_exception(error) diff --git a/lib/gitlab/ci/config/entry/include.rb b/lib/gitlab/ci/config/entry/include.rb index ad0ed00aa6f..368d8f07f8d 100644 --- a/lib/gitlab/ci/config/entry/include.rb +++ b/lib/gitlab/ci/config/entry/include.rb @@ -9,8 +9,10 @@ module Gitlab # class Include < ::Gitlab::Config::Entry::Node include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Configurable + include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[local file remote template artifact job project ref].freeze + ALLOWED_KEYS = %i[local file remote template artifact job project ref rules].freeze validations do validates :config, hash_or_string: true @@ -27,6 +29,20 @@ module Gitlab errors.add(:config, "must specify the file where to fetch the config from") end end + + with_options allow_nil: true do + validates :rules, array_of_hashes: true + end + end + + entry :rules, ::Gitlab::Ci::Config::Entry::Include::Rules, + description: 'List of evaluable Rules to determine file inclusion.', + inherit: false + + attributes :rules + + def skip_config_hash_validation? + true end end end diff --git a/lib/gitlab/ci/config/entry/include/rules.rb b/lib/gitlab/ci/config/entry/include/rules.rb new file mode 100644 index 00000000000..8eaf9e35aaf --- /dev/null +++ b/lib/gitlab/ci/config/entry/include/rules.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Include + class Rules < ::Gitlab::Config::Entry::ComposableArray + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, presence: true + validates :config, type: Array + end + + def value + @config + end + + def composable_class + Entry::Include::Rules::Rule + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/include/rules/rule.rb b/lib/gitlab/ci/config/entry/include/rules/rule.rb new file mode 100644 index 00000000000..d3d0f098814 --- /dev/null +++ b/lib/gitlab/ci/config/entry/include/rules/rule.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + class Include + class Rules::Rule < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + ALLOWED_KEYS = %i[if].freeze + + attributes :if + + validations do + validates :config, presence: true + validates :config, type: { with: Hash } + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :if, expression: true + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/inherit/variables.rb b/lib/gitlab/ci/config/entry/inherit/variables.rb index aa68833bdb8..adef4d1636a 100644 --- a/lib/gitlab/ci/config/entry/inherit/variables.rb +++ b/lib/gitlab/ci/config/entry/inherit/variables.rb @@ -13,9 +13,6 @@ module Gitlab strategy :ArrayStrategy, if: -> (config) { config.is_a?(Array) } class BooleanStrategy < ::Gitlab::Config::Entry::Boolean - def inherit?(_key) - value - end end class ArrayStrategy < ::Gitlab::Config::Entry::Node @@ -25,20 +22,12 @@ module Gitlab validates :config, type: Array validates :config, array_of_strings: true end - - def inherit?(key) - value.include?(key.to_s) - end end class UnknownStrategy < ::Gitlab::Config::Entry::Node def errors ["#{location} should be a bool or array of strings"] end - - def inherit?(key) - false - end end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index e6d63969161..bd4d5f33689 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -16,11 +16,8 @@ module Gitlab environment coverage retry parallel interruptible timeout release dast_configuration secrets].freeze - REQUIRED_BY_NEEDS = %i[stage].freeze - validations do validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS - validates :config, required_keys: REQUIRED_BY_NEEDS, if: :has_needs? validates :script, presence: true with_options allow_nil: true do diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 79dfb0eec1d..3543b5493bd 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -31,7 +31,7 @@ module Gitlab with_options allow_nil: true do validates :extends, array_of_strings_or_string: true - validates :rules, array_of_hashes: true + validates :rules, nested_array_of_hashes: true validates :resource_group, type: String end end @@ -88,9 +88,6 @@ module Gitlab validate_against_warnings end - # inherit root variables - @root_variables_value = deps&.variables_value # rubocop:disable Gitlab/ModuleWithInstanceVariables - yield if block_given? end end @@ -123,27 +120,13 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - variables: root_and_job_variables_value, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 - job_variables: job_variables, + job_variables: variables_value.to_h, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, resource_group: resource_group }.compact end - def root_and_job_variables_value - root_variables = @root_variables_value.to_h # rubocop:disable Gitlab/ModuleWithInstanceVariables - root_variables = root_variables.select do |key, _| - inherit_entry&.variables_entry&.inherit?(key) - end - - root_variables.merge(variables_value.to_h) - end - - def job_variables - variables_value.to_h - end - def root_variables_inheritance inherit_entry&.variables_entry&.value end diff --git a/lib/gitlab/ci/config/entry/rules.rb b/lib/gitlab/ci/config/entry/rules.rb index bf74f995e80..53e52981471 100644 --- a/lib/gitlab/ci/config/entry/rules.rb +++ b/lib/gitlab/ci/config/entry/rules.rb @@ -13,7 +13,7 @@ module Gitlab end def value - @config + [@config].flatten end def composable_class diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 567a86c47e5..4bd8e250d7a 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -45,7 +45,7 @@ module Gitlab errors.push("Remote file `#{location}` could not be fetched because of HTTP code `#{response.code}` error!") end - response.to_s if errors.none? + response.body if errors.none? end end end diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 3216d4eaac4..97e4922b2a1 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -33,6 +33,7 @@ module Gitlab locations .compact .map(&method(:normalize_location)) + .filter_map(&method(:verify_rules)) .flat_map(&method(:expand_project_files)) .flat_map(&method(:expand_wildcard_paths)) .map(&method(:expand_variables)) @@ -56,6 +57,15 @@ module Gitlab end end + def verify_rules(location) + # Behaves like there is no `rules` + return location unless ::Feature.enabled?(:ci_include_rules, context.project, default_enabled: :yaml) + + return unless Rules.new(location[:rules]).evaluate(context).pass? + + location + end + def expand_project_files(location) return location unless location[:project] @@ -65,8 +75,6 @@ module Gitlab end def expand_wildcard_paths(location) - return location unless ::Feature.enabled?(:ci_wildcard_file_paths, context.project, default_enabled: :yaml) - # We only support local files for wildcard paths return location unless location[:local] && location[:local].include?('*') diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb new file mode 100644 index 00000000000..5a788427172 --- /dev/null +++ b/lib/gitlab/ci/config/external/rules.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + class Rules + def initialize(rule_hashes) + @rule_list = Build::Rules::Rule.fabricate_list(rule_hashes) + end + + def evaluate(context) + Result.new(@rule_list.nil? || match_rule(context)) + end + + private + + def match_rule(context) + @rule_list.find { |rule| rule.matches?(nil, context) } + end + + Result = Struct.new(:result) do + def pass? + !!result + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb index 5cabbc86d3e..312f98f850a 100644 --- a/lib/gitlab/ci/config/normalizer/matrix_strategy.rb +++ b/lib/gitlab/ci/config/normalizer/matrix_strategy.rb @@ -43,7 +43,6 @@ module Gitlab { name: name, instance: instance, - variables: variables, # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 job_variables: variables, parallel: { total: total } }.compact diff --git a/lib/gitlab/ci/features.rb b/lib/gitlab/ci/features.rb index d26a903c1f8..51051b0490f 100644 --- a/lib/gitlab/ci/features.rb +++ b/lib/gitlab/ci/features.rb @@ -3,7 +3,7 @@ module Gitlab module Ci ## - # Ci::Features is a class that aggregates all CI/CD feature flags in one place. + # Deprecated: Ci::Features is a class that aggregates all CI/CD feature flags in one place. # module Features # NOTE: The feature flag `disallow_to_create_merge_request_pipelines_in_target_project` diff --git a/lib/gitlab/ci/limit.rb b/lib/gitlab/ci/limit.rb index c22a3c503d5..4f914388969 100644 --- a/lib/gitlab/ci/limit.rb +++ b/lib/gitlab/ci/limit.rb @@ -24,10 +24,13 @@ module Gitlab end def log_error!(extra_context = {}) - error = LimitExceededError.new(message) - # TODO: change this to Gitlab::ErrorTracking.log_exception(error, extra_context) - # https://gitlab.com/gitlab-org/gitlab/issues/32906 - ::Gitlab::ErrorTracking.track_exception(error, extra_context) + ::Gitlab::ErrorTracking.log_exception(limit_exceeded_error, extra_context) + end + + protected + + def limit_exceeded_error + LimitExceededError.new(message) end end end diff --git a/lib/gitlab/ci/lint.rb b/lib/gitlab/ci/lint.rb index 4a7c11ee26e..cd2c135dd7e 100644 --- a/lib/gitlab/ci/lint.rb +++ b/lib/gitlab/ci/lint.rb @@ -38,6 +38,7 @@ module Gitlab pipeline = ::Ci::CreatePipelineService .new(@project, @current_user, ref: @project.default_branch) .execute(:push, dry_run: true, content: content) + .payload Result.new( jobs: dry_run_convert_to_jobs(pipeline.stages), diff --git a/lib/gitlab/ci/model.rb b/lib/gitlab/ci/model.rb deleted file mode 100644 index 1625cb841b6..00000000000 --- a/lib/gitlab/ci/model.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Model - def table_name_prefix - "ci_" - end - - def model_name - @model_name ||= ActiveModel::Name.new(self, nil, self.name.demodulize) - end - end - end -end diff --git a/lib/gitlab/ci/parsers.rb b/lib/gitlab/ci/parsers.rb index 3469537a2e2..1223d664214 100644 --- a/lib/gitlab/ci/parsers.rb +++ b/lib/gitlab/ci/parsers.rb @@ -11,7 +11,9 @@ module Gitlab cobertura: ::Gitlab::Ci::Parsers::Coverage::Cobertura, terraform: ::Gitlab::Ci::Parsers::Terraform::Tfplan, accessibility: ::Gitlab::Ci::Parsers::Accessibility::Pa11y, - codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate + codequality: ::Gitlab::Ci::Parsers::Codequality::CodeClimate, + sast: ::Gitlab::Ci::Parsers::Security::Sast, + secret_detection: ::Gitlab::Ci::Parsers::Security::SecretDetection } end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb new file mode 100644 index 00000000000..41acb4d5040 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -0,0 +1,266 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class Common + SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError) + + def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse! + end + + def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + @json_data = json_data + @report = report + @validate = validate + @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + end + + def parse! + return report_data unless valid? + + raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) + + create_scanner + create_scan + create_analyzer + set_report_version + + create_findings + + report_data + rescue JSON::ParserError + raise SecurityReportParserError, 'JSON parsing failed' + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + raise SecurityReportParserError, "#{report.type} security report parsing failed" + end + + private + + attr_reader :json_data, :report, :validate + + def valid? + return true if !validate || schema_validator.valid? + + schema_validator.errors.each { |error| report.add_error('Schema', error) } + + false + end + + def schema_validator + @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data) + end + + def report_data + @report_data ||= Gitlab::Json.parse!(json_data) + end + + def report_version + @report_version ||= report_data['version'] + end + + def top_level_scanner + @top_level_scanner ||= report_data.dig('scan', 'scanner') + end + + def scan_data + @scan_data ||= report_data.dig('scan') + end + + def analyzer_data + @analyzer_data ||= report_data.dig('scan', 'analyzer') + end + + def tracking_data(data) + data['tracking'] + end + + def create_findings + if report_data["vulnerabilities"] + report_data["vulnerabilities"].each { |finding| create_finding(finding) } + end + end + + def create_finding(data, remediations = []) + identifiers = create_identifiers(data['identifiers']) + links = create_links(data['links']) + location = create_location(data['location'] || {}) + signatures = create_signatures(tracking_data(data)) + + if @vulnerability_finding_signatures_enabled && !signatures.empty? + # NOT the signature_sha - the compare key is hashed + # to create the project_fingerprint + highest_priority_signature = signatures.max_by(&:priority) + uuid = calculate_uuid_v5(identifiers.first, highest_priority_signature.signature_hex) + else + uuid = calculate_uuid_v5(identifiers.first, location&.fingerprint) + end + + report.add_finding( + ::Gitlab::Ci::Reports::Security::Finding.new( + uuid: uuid, + report_type: report.type, + name: finding_name(data, identifiers, location), + compare_key: data['cve'] || '', + location: location, + severity: parse_severity_level(data['severity']), + confidence: parse_confidence_level(data['confidence']), + scanner: create_scanner(data['scanner']), + scan: report&.scan, + identifiers: identifiers, + links: links, + remediations: remediations, + raw_metadata: data.to_json, + metadata_version: report_version, + details: data['details'] || {}, + signatures: signatures, + project_id: report.project_id, + vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) + end + + def create_signatures(tracking) + tracking ||= { 'items' => [] } + + signature_algorithms = Hash.new { |hash, key| hash[key] = [] } + + tracking['items'].each do |item| + next unless item.key?('signatures') + + item['signatures'].each do |signature| + alg = signature['algorithm'] + signature_algorithms[alg] << signature['value'] + end + end + + signature_algorithms.map do |algorithm, values| + value = values.join('|') + signature = ::Gitlab::Ci::Reports::Security::FindingSignature.new( + algorithm_type: algorithm, + signature_value: value + ) + + if signature.valid? + signature + else + e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}") + Gitlab::ErrorTracking.track_exception(e) + nil + end + end.compact + end + + def create_scan + return unless scan_data.is_a?(Hash) + + report.scan = ::Gitlab::Ci::Reports::Security::Scan.new(scan_data) + end + + def set_report_version + report.version = report_version + end + + def create_analyzer + return unless analyzer_data.is_a?(Hash) + + params = { + id: analyzer_data.dig('id'), + name: analyzer_data.dig('name'), + version: analyzer_data.dig('version'), + vendor: analyzer_data.dig('vendor', 'name') + } + + return unless params.values.all? + + report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params) + end + + def create_scanner(scanner_data = top_level_scanner) + return unless scanner_data.is_a?(Hash) + + report.add_scanner( + ::Gitlab::Ci::Reports::Security::Scanner.new( + external_id: scanner_data['id'], + name: scanner_data['name'], + vendor: scanner_data.dig('vendor', 'name'), + version: scanner_data.dig('version'))) + end + + def create_identifiers(identifiers) + return [] unless identifiers.is_a?(Array) + + identifiers.map { |identifier| create_identifier(identifier) }.compact + end + + def create_identifier(identifier) + return unless identifier.is_a?(Hash) + + report.add_identifier( + ::Gitlab::Ci::Reports::Security::Identifier.new( + external_type: identifier['type'], + external_id: identifier['value'], + name: identifier['name'], + url: identifier['url'])) + end + + def create_links(links) + return [] unless links.is_a?(Array) + + links.map { |link| create_link(link) }.compact + end + + def create_link(link) + return unless link.is_a?(Hash) + + ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url']) + end + + def parse_severity_level(input) + input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' } + end + + def parse_confidence_level(input) + input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' } + end + + def create_location(location_data) + raise NotImplementedError + end + + def finding_name(data, identifiers, location) + return data['message'] if data['message'].present? + return data['name'] if data['name'].present? + + identifier = identifiers.find(&:cve?) || identifiers.find(&:cwe?) || identifiers.first + "#{identifier.name} in #{location&.fingerprint_path}" + end + + def calculate_uuid_v5(primary_identifier, location_fingerprint) + uuid_v5_name_components = { + report_type: report.type, + primary_identifier_fingerprint: primary_identifier&.fingerprint, + location_fingerprint: location_fingerprint, + project_id: report.project_id + } + + if uuid_v5_name_components.values.any?(&:nil?) + Gitlab::AppLogger.warn(message: "One or more UUID name components are nil", components: uuid_v5_name_components) + return + end + + ::Security::VulnerabilityUUID.generate( + report_type: uuid_v5_name_components[:report_type], + primary_identifier_fingerprint: uuid_v5_name_components[:primary_identifier_fingerprint], + location_fingerprint: uuid_v5_name_components[:location_fingerprint], + project_id: uuid_v5_name_components[:project_id] + ) + end + end + end + end + end +end + +Gitlab::Ci::Parsers::Security::Common.prepend_mod_with("Gitlab::Ci::Parsers::Security::Common") diff --git a/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb new file mode 100644 index 00000000000..24613a441be --- /dev/null +++ b/lib/gitlab/ci/parsers/security/concerns/deprecated_syntax.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + module Concerns + module DeprecatedSyntax + extend ActiveSupport::Concern + + included do + extend ::Gitlab::Utils::Override + + override :parse_report + end + + def report_data + @report_data ||= begin + data = super + + if data.is_a?(Array) + data = { + "version" => self.class::DEPRECATED_REPORT_VERSION, + "vulnerabilities" => data + } + end + + data + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/sast.rb b/lib/gitlab/ci/parsers/security/sast.rb new file mode 100644 index 00000000000..e3c62614cd8 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/sast.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class Sast < Common + include Security::Concerns::DeprecatedSyntax + + DEPRECATED_REPORT_VERSION = "1.2" + + private + + def create_location(location_data) + ::Gitlab::Ci::Reports::Security::Locations::Sast.new( + file_path: location_data['file'], + start_line: location_data['start_line'], + end_line: location_data['end_line'], + class_name: location_data['class'], + method_name: location_data['method']) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/secret_detection.rb b/lib/gitlab/ci/parsers/security/secret_detection.rb new file mode 100644 index 00000000000..c6d95c1d391 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/secret_detection.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class SecretDetection < Common + include Security::Concerns::DeprecatedSyntax + + DEPRECATED_REPORT_VERSION = "1.2" + + private + + def create_location(location_data) + ::Gitlab::Ci::Reports::Security::Locations::SecretDetection.new( + file_path: location_data['file'], + start_line: location_data['start_line'], + end_line: location_data['end_line'], + class_name: location_data['class'], + method_name: location_data['method'] + ) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb new file mode 100644 index 00000000000..3d92886cba8 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + module Validators + class SchemaValidator + class Schema + def root_path + File.join(__dir__, 'schemas') + end + + def initialize(report_type) + @report_type = report_type + end + + delegate :validate, to: :schemer + + private + + attr_reader :report_type + + def schemer + JSONSchemer.schema(pathname) + end + + def pathname + Pathname.new(schema_path) + end + + def schema_path + File.join(root_path, file_name) + end + + def file_name + "#{report_type}.json" + end + end + + def initialize(report_type, report_data) + @report_type = report_type + @report_data = report_data + end + + def valid? + errors.empty? + end + + def errors + @errors ||= schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + end + + private + + attr_reader :report_type, :report_data + + def schema + Schema.new(report_type) + end + end + end + end + end + end +end + +Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema.prepend_mod_with("Gitlab::Ci::Parsers::Security::Validators::SchemaValidator::Schema") diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/sast.json b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json new file mode 100644 index 00000000000..a7159be0190 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/sast.json @@ -0,0 +1,706 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.0.0" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "type": "object", + "description": "The vendor/maintainer of the scanner.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability.", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json new file mode 100644 index 00000000000..462e23a151c --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/secret_detection.json @@ -0,0 +1,729 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.0.0" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "type": "object", + "description": "The vendor/maintainer of the scanner.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability.", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 7564d0c3ed5..626eba97817 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -97,15 +97,16 @@ module Gitlab .observe({ source: pipeline.source.to_s }, pipeline.total_size) end + def observe_jobs_count_in_alive_pipelines + metrics.active_jobs_histogram + .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines) + end + def increment_pipeline_failure_reason_counter(reason) metrics.pipeline_failure_reason_counter .increment(reason: (reason || :unknown_failure).to_s) end - def dangling_build? - %i[ondemand_dast_scan webide].include?(source) - end - private # Verifies that origin_ref is a fully qualified tag reference (refs/tags/<tag-name>) diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index 49ec1250a5f..5251dd3d40a 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -14,7 +14,7 @@ module Gitlab result = ::Gitlab::Ci::YamlProcessor.new( @command.config_content, { project: project, - ref: @pipeline.ref, + source_ref_path: @pipeline.source_ref_path, sha: @pipeline.sha, source: @pipeline.source, user: current_user, diff --git a/lib/gitlab/ci/pipeline/chain/sequence.rb b/lib/gitlab/ci/pipeline/chain/sequence.rb index dc648568129..bbfc6759b35 100644 --- a/lib/gitlab/ci/pipeline/chain/sequence.rb +++ b/lib/gitlab/ci/pipeline/chain/sequence.rb @@ -22,6 +22,7 @@ module Gitlab @command.observe_creation_duration(Time.now - @start) @command.observe_pipeline_size(@pipeline) + @command.observe_jobs_count_in_alive_pipelines @pipeline end diff --git a/lib/gitlab/ci/pipeline/chain/skip.rb b/lib/gitlab/ci/pipeline/chain/skip.rb index e4e4f4f484a..76dfb4cbd87 100644 --- a/lib/gitlab/ci/pipeline/chain/skip.rb +++ b/lib/gitlab/ci/pipeline/chain/skip.rb @@ -22,16 +22,16 @@ module Gitlab end end - def skipped? - !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) - end - def break? skipped? end private + def skipped? + !@command.ignore_skip_ci && (commit_message_skips_ci? || push_option_skips_ci?) + end + def commit_message_skips_ci? return false unless @pipeline.git_commit_message diff --git a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb index 514241e8ae2..c7106f3ec39 100644 --- a/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb +++ b/lib/gitlab/ci/pipeline/expression/lexeme/pattern.rb @@ -11,7 +11,7 @@ module Gitlab PATTERN = %r{^\/([^\/]|\\/)+[^\\]\/[ismU]*}.freeze def initialize(regexp) - super(regexp.gsub(/\\\//, '/')) + super(regexp.gsub(%r{\\/}, '/')) unless Gitlab::UntrustedRegexp::RubySyntax.valid?(@value) raise Lexer::SyntaxError, 'Invalid regular expression!' diff --git a/lib/gitlab/ci/pipeline/metrics.rb b/lib/gitlab/ci/pipeline/metrics.rb index 84b88374a7f..10de77afe74 100644 --- a/lib/gitlab/ci/pipeline/metrics.rb +++ b/lib/gitlab/ci/pipeline/metrics.rb @@ -24,7 +24,16 @@ module Gitlab name = :gitlab_ci_pipeline_size_builds comment = 'Pipeline size' labels = { source: nil } - buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000] + buckets = [0, 1, 5, 10, 20, 50, 100, 200, 500, 1000, 2000, 3000] + + ::Gitlab::Metrics.histogram(name, comment, labels, buckets) + end + + def self.active_jobs_histogram + name = :gitlab_ci_active_jobs + comment = 'Total amount of active jobs' + labels = { plan: nil } + buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000] ::Gitlab::Metrics.histogram(name, comment, labels, buckets) end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 54d92745992..c393fed26de 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -39,7 +39,7 @@ module Gitlab @cache = Gitlab::Ci::Build::Cache .new(attributes.delete(:cache), @pipeline) - recalculate_yaml_variables! + calculate_yaml_variables! end def name @@ -232,7 +232,7 @@ module Gitlab { options: { allow_failure_criteria: nil } } end - def recalculate_yaml_variables! + def calculate_yaml_variables! @seed_attributes[:yaml_variables] = Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: @context.root_variables, to: @job_variables, inheritance: @root_variables_inheritance ) diff --git a/lib/gitlab/ci/reports/security/aggregated_report.rb b/lib/gitlab/ci/reports/security/aggregated_report.rb new file mode 100644 index 00000000000..a8bb2196043 --- /dev/null +++ b/lib/gitlab/ci/reports/security/aggregated_report.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Used to represent combined Security Reports. This is typically done for vulnerability deduplication purposes. + +module Gitlab + module Ci + module Reports + module Security + class AggregatedReport + attr_reader :findings + + def initialize(reports, findings) + @reports = reports + @findings = findings + end + + def created_at + @reports.map(&:created_at).compact.min + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb new file mode 100644 index 00000000000..dc1c51b3ed0 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Finding + include ::VulnerabilityFindingHelpers + + attr_reader :compare_key + attr_reader :confidence + attr_reader :identifiers + attr_reader :links + attr_reader :location + attr_reader :metadata_version + attr_reader :name + attr_reader :old_location + attr_reader :project_fingerprint + attr_reader :raw_metadata + attr_reader :report_type + attr_reader :scanner + attr_reader :scan + attr_reader :severity + attr_accessor :uuid + attr_accessor :overridden_uuid + attr_reader :remediations + attr_reader :details + attr_reader :signatures + attr_reader :project_id + + delegate :file_path, :start_line, :end_line, to: :location + + def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + @compare_key = compare_key + @confidence = confidence + @identifiers = identifiers + @links = links + @location = location + @metadata_version = metadata_version + @name = name + @raw_metadata = raw_metadata + @report_type = report_type + @scanner = scanner + @scan = scan + @severity = severity + @uuid = uuid + @remediations = remediations + @details = details + @signatures = signatures + @project_id = project_id + @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + + @project_fingerprint = generate_project_fingerprint + end + + def to_hash + %i[ + compare_key + confidence + identifiers + links + location + metadata_version + name + project_fingerprint + raw_metadata + report_type + scanner + scan + severity + uuid + details + signatures + ].each_with_object({}) do |key, hash| + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def primary_identifier + identifiers.first + end + + def update_location(new_location) + @old_location = location + @location = new_location + end + + def unsafe?(severity_levels) + severity.in?(severity_levels) + end + + def eql?(other) + return false unless report_type == other.report_type && primary_identifier_fingerprint == other.primary_identifier_fingerprint + + if @vulnerability_finding_signatures_enabled + matches_signatures(other.signatures, other.uuid) + else + location.fingerprint == other.location.fingerprint + end + end + + def hash + if @vulnerability_finding_signatures_enabled && !signatures.empty? + highest_signature = signatures.max_by(&:priority) + report_type.hash ^ highest_signature.signature_hex.hash ^ primary_identifier_fingerprint.hash + else + report_type.hash ^ location.fingerprint.hash ^ primary_identifier_fingerprint.hash + end + end + + def valid? + scanner.present? && primary_identifier.present? && location.present? && uuid.present? + end + + def keys + @keys ||= identifiers.reject(&:type_identifier?).map do |identifier| + FindingKey.new(location_fingerprint: location&.fingerprint, identifier_fingerprint: identifier.fingerprint) + end + end + + def primary_identifier_fingerprint + primary_identifier&.fingerprint + end + + def <=>(other) + if severity == other.severity + compare_key <=> other.compare_key + else + ::Enums::Vulnerability.severity_levels[other.severity] <=> + ::Enums::Vulnerability.severity_levels[severity] + end + end + + def scanner_order_to(other) + return 1 unless scanner + return -1 unless other&.scanner + + scanner <=> other.scanner + end + + private + + def generate_project_fingerprint + Digest::SHA1.hexdigest(compare_key) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb new file mode 100644 index 00000000000..0acd923a60f --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_key.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingKey + def initialize(location_fingerprint:, identifier_fingerprint:) + @location_fingerprint = location_fingerprint + @identifier_fingerprint = identifier_fingerprint + end + + def ==(other) + has_fingerprints? && other.has_fingerprints? && + location_fingerprint == other.location_fingerprint && + identifier_fingerprint == other.identifier_fingerprint + end + + def hash + location_fingerprint.hash ^ identifier_fingerprint.hash + end + + alias_method :eql?, :== + + protected + + attr_reader :location_fingerprint, :identifier_fingerprint + + def has_fingerprints? + location_fingerprint.present? && identifier_fingerprint.present? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_signature.rb b/lib/gitlab/ci/reports/security/finding_signature.rb new file mode 100644 index 00000000000..d1d7ef5c377 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_signature.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingSignature + include VulnerabilityFindingSignatureHelpers + + attr_accessor :algorithm_type, :signature_value + + def initialize(params = {}) + @algorithm_type = params.dig(:algorithm_type) + @signature_value = params.dig(:signature_value) + end + + def signature_sha + Digest::SHA1.digest(signature_value) + end + + def signature_hex + signature_sha.unpack1("H*") + end + + def to_hash + { + algorithm_type: algorithm_type, + signature_sha: signature_sha + } + end + + def valid? + algorithm_types.key?(algorithm_type) + end + + def eql?(other) + other.algorithm_type == algorithm_type && + other.signature_sha == signature_sha + end + + alias_method :==, :eql? + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/base.rb b/lib/gitlab/ci/reports/security/locations/base.rb new file mode 100644 index 00000000000..9ad1d81287f --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/base.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Base + include ::Gitlab::Utils::StrongMemoize + + def ==(other) + other.fingerprint == fingerprint + end + + def fingerprint + strong_memoize(:fingerprint) do + Digest::SHA1.hexdigest(fingerprint_data) + end + end + + def as_json(options = nil) + fingerprint # side-effect call to initialize the ivar for serialization + + super + end + + def fingerprint_path + fingerprint_data + end + + private + + def fingerprint_data + raise NotImplementedError + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/sast.rb b/lib/gitlab/ci/reports/security/locations/sast.rb new file mode 100644 index 00000000000..23ffa91e720 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/sast.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Sast < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/secret_detection.rb b/lib/gitlab/ci/reports/security/locations/secret_detection.rb new file mode 100644 index 00000000000..0fd5cc5af11 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/secret_detection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class SecretDetection < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb new file mode 100644 index 00000000000..1ba2d909d99 --- /dev/null +++ b/lib/gitlab/ci/reports/security/report.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Report + attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version + + delegate :project_id, to: :pipeline + + def initialize(type, pipeline, created_at) + @type = type + @pipeline = pipeline + @created_at = created_at + @findings = [] + @scanners = {} + @identifiers = {} + @scanned_resources = [] + @errors = [] + end + + def commit_sha + pipeline.sha + end + + def add_error(type, message = 'An unexpected error happened!') + errors << { type: type, message: message } + end + + def errored? + errors.present? + end + + def add_scanner(scanner) + scanners[scanner.key] ||= scanner + end + + def add_identifier(identifier) + identifiers[identifier.key] ||= identifier + end + + def add_finding(finding) + findings << finding + end + + def clone_as_blank + Report.new(type, pipeline, created_at) + end + + def replace_with!(other) + instance_variables.each do |ivar| + instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def merge!(other) + replace_with!(::Security::MergeReportsService.new(self, other).execute) + end + + def primary_scanner + scanners.first&.second + end + + def primary_scanner_order_to(other) + return 1 unless primary_scanner + return -1 unless other.primary_scanner + + primary_scanner <=> other.primary_scanner + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb new file mode 100644 index 00000000000..b7a5e36b108 --- /dev/null +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Reports + attr_reader :reports, :pipeline + + delegate :each, :empty?, to: :reports + + def initialize(pipeline) + @reports = {} + @pipeline = pipeline + end + + def get_report(report_type, report_artifact) + reports[report_type] ||= Report.new(report_type, pipeline, report_artifact.created_at) + end + + def findings + reports.values.flat_map(&:findings) + end + + def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels) + unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed + end + + private + + def findings_diff(target_reports) + findings - target_reports&.findings.to_a + end + + def unsafe_findings_count(target_reports, severity_levels) + findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb new file mode 100644 index 00000000000..6cb2e0ddb33 --- /dev/null +++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class VulnerabilityReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_report, :head_report + + ACCEPTABLE_REPORT_AGE = 1.week + + def initialize(project, base_report, head_report) + @base_report = base_report + @head_report = head_report + + @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures) + + if @signatures_enabled + @added_findings = [] + @fixed_findings = [] + calculate_changes + end + end + + def base_report_created_at + @base_report.created_at + end + + def head_report_created_at + @head_report.created_at + end + + def base_report_out_of_date + return false unless @base_report.created_at + + ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at + end + + def added + strong_memoize(:added) do + if @signatures_enabled + @added_findings + else + head_report.findings - base_report.findings + end + end + end + + def fixed + strong_memoize(:fixed) do + if @signatures_enabled + @fixed_findings + else + base_report.findings - head_report.findings + end + end + end + + private + + def calculate_changes + # This is a deconstructed version of the eql? method on + # Ci::Reports::Security::Finding. It: + # + # * precomputes for the head_findings (using FindingMatcher): + # * sets of signature shas grouped by priority + # * mappings of signature shas to the head finding object + # + # These are then used when iterating the base findings to perform + # fast(er) prioritized, signature-based comparisons between each base finding + # and the head findings. + # + # Both the head_findings and base_findings arrays are iterated once + + base_findings = base_report.findings + head_findings = head_report.findings + + matcher = FindingMatcher.new(head_findings) + + base_findings.each do |base_finding| + matched_head_finding = matcher.find_and_remove_match!(base_finding) + + @fixed_findings << base_finding if matched_head_finding.nil? + end + + @added_findings = matcher.unmatched_head_findings.values + end + end + + class FindingMatcher + attr_reader :unmatched_head_findings, :head_findings + + include Gitlab::Utils::StrongMemoize + + def initialize(head_findings) + @head_findings = head_findings + @unmatched_head_findings = @head_findings.index_by(&:object_id) + end + + def find_and_remove_match!(base_finding) + matched_head_finding = find_matched_head_finding_for(base_finding) + + # no signatures matched, so check the normal uuids of the base and head findings + # for a match + matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil? + + @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil? + + matched_head_finding + end + + private + + def find_matched_head_finding_for(base_finding) + base_signature = sorted_signatures_for(base_finding).find do |signature| + # at this point a head_finding exists that has a signature with a + # matching priority, and a matching sha --> lookup the actual finding + # object from head_signatures_shas + head_signatures_shas[signature.signature_sha].eql?(base_finding) + end + + base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil + end + + def sorted_signatures_for(base_finding) + base_finding.signatures.select { |signature| head_finding_signature?(signature) } + .sort_by { |sig| -sig.priority } + end + + def head_finding_signature?(signature) + head_signatures_priorities[signature.priority].include?(signature.signature_sha) + end + + def head_signatures_priorities + strong_memoize(:head_signatures_priorities) do + signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new } + + head_findings.each_with_object(signatures_priorities) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.priority].add(signature.signature_sha) + end + end + end + end + + def head_signatures_shas + strong_memoize(:head_signatures_shas) do + head_findings.each_with_object({}) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.signature_sha] = head_finding + end + # for the final uuid check when no signatures have matched + memo[head_finding.uuid] = head_finding + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml index ebb0b5948f1..71f38ededd9 100644 --- a/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/5-Minute-Production-App.gitlab-ci.yml @@ -5,7 +5,7 @@ # This template is on early stage of development. # Use it with caution. For usage instruction please read -# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v2.3.0/README.md +# https://gitlab.com/gitlab-org/5-minute-production-app/deploy-template/-/blob/v3.0.0/README.md include: # workflow rules to prevent duplicate detached pipelines diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 1910913f2bd..f39a84bceec 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Bash.gitlab-ci.yml -# See https://docs.gitlab.com/ee/ci/yaml/README.html for all available options +# See https://docs.gitlab.com/ee/ci/yaml/index.html for all available options # you can delete this line if you're not using Docker image: busybox:latest diff --git a/lib/gitlab/ci/templates/Django.gitlab-ci.yml b/lib/gitlab/ci/templates/Django.gitlab-ci.yml index d2d3b3ed61e..f147ad9332d 100644 --- a/lib/gitlab/ci/templates/Django.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Django.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: POSTGRES_DB: database_name # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - ~/.cache/pip/ diff --git a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml index 38036c1f964..21a599fc78d 100644 --- a/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml @@ -10,7 +10,7 @@ # A pipeline is composed of independent jobs that run scripts, grouped into stages. # Stages run in sequential order, but jobs within stages run in parallel. # -# For more information, see: https://docs.gitlab.com/ee/ci/yaml/README.html#stages +# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages stages: # List of stages for jobs, and their order of execution - build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 48e877684f6..43ecc4b96d5 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -27,7 +27,7 @@ code_quality: } - docker pull --quiet "$CODE_QUALITY_IMAGE" - | - docker run \ + docker run --rm \ $(propagate_env_vars \ SOURCE_CODE \ TIMEOUT_SECONDS \ diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 00fcfa64a18..208951fa1a1 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .dast-auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" dast_environment_deploy: extends: .dast-auto-deploy diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 530ab1d0f99..5c466f0984c 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ .auto-deploy: - image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.6.0" + image: "registry.gitlab.com/gitlab-org/cluster-integration/auto-deploy-image:v2.12.0" dependencies: [] review: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 80125a9bc01..917a28bb1ee 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -252,6 +252,7 @@ semgrep-sast: - '**/*.jsx' - '**/*.ts' - '**/*.tsx' + - '**/*.c' sobelow-sast: extends: .sast-analyzer diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index d0595491400..18f0f20203d 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -27,8 +27,8 @@ secret_detection: when: never - if: $CI_COMMIT_BRANCH script: - - if [[ $CI_COMMIT_TAG ]]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi - - if [[ $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH ]]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi + - if [ -n "$CI_COMMIT_TAG" ]; then echo "Skipping Secret Detection for tags. No code changes have occurred."; exit 0; fi + - if [ "$CI_COMMIT_BRANCH" = "$CI_DEFAULT_BRANCH" ]; then echo "Running Secret Detection on default branch."; /analyzer run; exit 0; fi - git fetch origin $CI_DEFAULT_BRANCH $CI_COMMIT_REF_NAME - git log --left-right --cherry-pick --pretty=format:"%H" refs/remotes/origin/$CI_DEFAULT_BRANCH...refs/remotes/origin/$CI_COMMIT_REF_NAME > "$CI_COMMIT_SHA"_commit_list.txt - export SECRET_DETECTION_COMMITS_FILE="$CI_COMMIT_SHA"_commit_list.txt diff --git a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml index 43e4ac02d41..ff7bac15017 100644 --- a/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Laravel.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: MYSQL_ROOT_PASSWORD: secret # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - vendor/ diff --git a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml index e48801b7970..16bc0026aa8 100644 --- a/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Nodejs.gitlab-ci.yml @@ -16,7 +16,7 @@ services: - postgres:latest # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml index d3726fe34c5..9da50439be8 100644 --- a/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Pages/Gatsby.gitlab-ci.yml @@ -6,7 +6,7 @@ image: node:latest # This folder is cached between builds -# http://docs.gitlab.com/ee/ci/yaml/README.html#cache +# https://docs.gitlab.com/ee/ci/yaml/index.html#cache cache: paths: - node_modules/ diff --git a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml index 490fc779e17..0c8b98dc1cf 100644 --- a/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Ruby.gitlab-ci.yml @@ -29,7 +29,8 @@ before_script: - ruby -v # Print out ruby version for debugging # Uncomment next line if your rails app needs a JS runtime: # - apt-get update -q && apt-get install nodejs -yqq - - bundle install -j $(nproc) --path vendor # Install dependencies into ./vendor/ruby + - bundle config set path 'vendor' # Install dependencies into ./vendor/ruby + - bundle install -j $(nproc) # Optional - Delete if not using `rubocop` rubocop: diff --git a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml index f4f066cc7c2..ed4876c2bcc 100644 --- a/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Cluster-Image-Scanning.gitlab-ci.yml @@ -8,7 +8,7 @@ # - A `test` stage to be present in the pipeline. # - You must define the `CIS_KUBECONFIG` variable to allow analyzer to connect to your Kubernetes cluster and fetch found vulnerabilities. # -# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/README.html). +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/#available-variables variables: diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml new file mode 100644 index 00000000000..d27a08db181 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -0,0 +1,23 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml + +stages: + - build + - test + - deploy + - dast + +variables: + DAST_RUNNER_VALIDATION_VERSION: 1 + +validation: + stage: dast + image: + name: "registry.gitlab.com/security-products/dast-runner-validation:$DAST_RUNNER_VALIDATION_VERSION" + variables: + GIT_STRATEGY: none + allow_failure: false + script: + - ~/validate.sh diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index e30777d8401..86b7d57d3cb 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -18,7 +18,7 @@ variables: bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, secrets, sobelow, pmd-apex, kubesec, semgrep, bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, license-finder, - dast, api-fuzzing + dast, dast-runner-validation, api-fuzzing SECURE_BINARIES_DOWNLOAD_IMAGES: "true" SECURE_BINARIES_PUSH_IMAGES: "true" @@ -230,6 +230,16 @@ dast: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ +dast-runner-validation: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "1" + SECURE_BINARIES_IMAGE: "registry.gitlab.com/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bdast-runner-validation\b/ + api-fuzzing: extends: .download_images variables: diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 272b980b4b2..1a857ef3eb3 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -4,7 +4,7 @@ # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml include: - - template: Terraform/Base.latest.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml + - template: Terraform/Base.gitlab-ci.yml # https://gitlab.com/gitlab-org/gitlab/blob/master/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml stages: - init diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index d34a847f2d5..a9f6fd88d0b 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -14,15 +14,22 @@ stages: - cleanup init: - extends: .init + extends: .terraform:init validate: - extends: .validate + extends: .terraform:validate build: - extends: .build + extends: .terraform:build deploy: - extends: .deploy + extends: .terraform:deploy dependencies: - build + environment: + name: $TF_STATE_NAME + +cleanup: + extends: .terraform:destroy + dependencies: + - deploy diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml new file mode 100644 index 00000000000..39c3374e534 --- /dev/null +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -0,0 +1,64 @@ +# Terraform/Base.latest +# +# The purpose of this template is to provide flexibility to the user so +# they are able to only include the jobs that they find interesting. +# +# Therefore, this template is not supposed to run any jobs. The idea is to only +# create hidden jobs. See: https://docs.gitlab.com/ee/ci/yaml/#hide-jobs +# +# There is a more opinionated template which we suggest the users to abide, +# which is the lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml + +image: + name: registry.gitlab.com/gitlab-org/terraform-images/releases/terraform:1.0.3 + +variables: + TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project + TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + +cache: + key: "${TF_ROOT}" + paths: + - ${TF_ROOT}/.terraform/ + - ${TF_ROOT}/.terraform.lock.hcl + +.init: &init + stage: init + script: + - cd ${TF_ROOT} + - gitlab-terraform init + +.validate: &validate + stage: validate + script: + - cd ${TF_ROOT} + - gitlab-terraform validate + +.build: &build + stage: build + script: + - cd ${TF_ROOT} + - gitlab-terraform plan + - gitlab-terraform plan-json + artifacts: + paths: + - ${TF_ROOT}/plan.cache + reports: + terraform: ${TF_ROOT}/plan.json + +.deploy: &deploy + stage: deploy + script: + - cd ${TF_ROOT} + - gitlab-terraform apply + when: manual + only: + variables: + - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + +.destroy: &destroy + stage: cleanup + script: + - cd ${TF_ROOT} + - gitlab-terraform destroy + when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 200388a274c..c30860ad174 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -13,7 +13,8 @@ image: name: registry.gitlab.com/gitlab-org/terraform-images/stable:latest variables: - TF_ROOT: ${CI_PROJECT_DIR} + TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project + TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" @@ -21,43 +22,46 @@ cache: - ${TF_ROOT}/.terraform/ - ${TF_ROOT}/.terraform.lock.hcl -.init: &init +.terraform:init: &terraform_init stage: init script: - cd ${TF_ROOT} - gitlab-terraform init -.validate: &validate +.terraform:validate: &terraform_validate stage: validate script: - cd ${TF_ROOT} - gitlab-terraform validate -.build: &build +.terraform:build: &terraform_build stage: build script: - cd ${TF_ROOT} - gitlab-terraform plan - gitlab-terraform plan-json + resource_group: ${TF_STATE_NAME} artifacts: paths: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json -.deploy: &deploy +.terraform:deploy: &terraform_deploy stage: deploy script: - cd ${TF_ROOT} - gitlab-terraform apply + resource_group: ${TF_STATE_NAME} when: manual only: variables: - $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH -.destroy: &destroy +.terraform:destroy: &terraform_destroy stage: cleanup script: - cd ${TF_ROOT} - gitlab-terraform destroy + resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 0140218d9bc..8ab9573dd20 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -23,7 +23,7 @@ module Gitlab new(nodes).tsort rescue TSort::Cyclic - raise ValidationError, 'The pipeline has circular dependencies.' + raise ValidationError, 'The pipeline has circular dependencies' rescue MissingNodeError end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index dd5107bad9a..a97c7050fbb 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -69,7 +69,7 @@ module Gitlab when: job[:when] || 'on_success', environment: job[:environment_name], coverage_regex: job[:coverage], - yaml_variables: transform_to_yaml_variables(job[:variables]), # https://gitlab.com/gitlab-org/gitlab/-/issues/300581 + # yaml_variables is calculated with using job_variables in Seed::Build job_variables: transform_to_yaml_variables(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), |