diff options
Diffstat (limited to 'lib/gitlab/ci/reports')
| -rw-r--r-- | lib/gitlab/ci/reports/security/aggregated_report.rb | 24 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/finding.rb | 150 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/finding_key.rb | 36 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/finding_signature.rb | 46 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/locations/base.rb | 41 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/locations/sast.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/locations/secret_detection.rb | 33 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/report.rb | 76 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/reports.rb | 42 | ||||
| -rw-r--r-- | lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb | 163 |
10 files changed, 644 insertions, 0 deletions
diff --git a/lib/gitlab/ci/reports/security/aggregated_report.rb b/lib/gitlab/ci/reports/security/aggregated_report.rb new file mode 100644 index 00000000000..a8bb2196043 --- /dev/null +++ b/lib/gitlab/ci/reports/security/aggregated_report.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +# Used to represent combined Security Reports. This is typically done for vulnerability deduplication purposes. + +module Gitlab + module Ci + module Reports + module Security + class AggregatedReport + attr_reader :findings + + def initialize(reports, findings) + @reports = reports + @findings = findings + end + + def created_at + @reports.map(&:created_at).compact.min + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb new file mode 100644 index 00000000000..dc1c51b3ed0 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -0,0 +1,150 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Finding + include ::VulnerabilityFindingHelpers + + attr_reader :compare_key + attr_reader :confidence + attr_reader :identifiers + attr_reader :links + attr_reader :location + attr_reader :metadata_version + attr_reader :name + attr_reader :old_location + attr_reader :project_fingerprint + attr_reader :raw_metadata + attr_reader :report_type + attr_reader :scanner + attr_reader :scan + attr_reader :severity + attr_accessor :uuid + attr_accessor :overridden_uuid + attr_reader :remediations + attr_reader :details + attr_reader :signatures + attr_reader :project_id + + delegate :file_path, :start_line, :end_line, to: :location + + def initialize(compare_key:, identifiers:, links: [], remediations: [], location:, metadata_version:, name:, raw_metadata:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + @compare_key = compare_key + @confidence = confidence + @identifiers = identifiers + @links = links + @location = location + @metadata_version = metadata_version + @name = name + @raw_metadata = raw_metadata + @report_type = report_type + @scanner = scanner + @scan = scan + @severity = severity + @uuid = uuid + @remediations = remediations + @details = details + @signatures = signatures + @project_id = project_id + @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + + @project_fingerprint = generate_project_fingerprint + end + + def to_hash + %i[ + compare_key + confidence + identifiers + links + location + metadata_version + name + project_fingerprint + raw_metadata + report_type + scanner + scan + severity + uuid + details + signatures + ].each_with_object({}) do |key, hash| + hash[key] = public_send(key) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def primary_identifier + identifiers.first + end + + def update_location(new_location) + @old_location = location + @location = new_location + end + + def unsafe?(severity_levels) + severity.in?(severity_levels) + end + + def eql?(other) + return false unless report_type == other.report_type && primary_identifier_fingerprint == other.primary_identifier_fingerprint + + if @vulnerability_finding_signatures_enabled + matches_signatures(other.signatures, other.uuid) + else + location.fingerprint == other.location.fingerprint + end + end + + def hash + if @vulnerability_finding_signatures_enabled && !signatures.empty? + highest_signature = signatures.max_by(&:priority) + report_type.hash ^ highest_signature.signature_hex.hash ^ primary_identifier_fingerprint.hash + else + report_type.hash ^ location.fingerprint.hash ^ primary_identifier_fingerprint.hash + end + end + + def valid? + scanner.present? && primary_identifier.present? && location.present? && uuid.present? + end + + def keys + @keys ||= identifiers.reject(&:type_identifier?).map do |identifier| + FindingKey.new(location_fingerprint: location&.fingerprint, identifier_fingerprint: identifier.fingerprint) + end + end + + def primary_identifier_fingerprint + primary_identifier&.fingerprint + end + + def <=>(other) + if severity == other.severity + compare_key <=> other.compare_key + else + ::Enums::Vulnerability.severity_levels[other.severity] <=> + ::Enums::Vulnerability.severity_levels[severity] + end + end + + def scanner_order_to(other) + return 1 unless scanner + return -1 unless other&.scanner + + scanner <=> other.scanner + end + + private + + def generate_project_fingerprint + Digest::SHA1.hexdigest(compare_key) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_key.rb b/lib/gitlab/ci/reports/security/finding_key.rb new file mode 100644 index 00000000000..0acd923a60f --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_key.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingKey + def initialize(location_fingerprint:, identifier_fingerprint:) + @location_fingerprint = location_fingerprint + @identifier_fingerprint = identifier_fingerprint + end + + def ==(other) + has_fingerprints? && other.has_fingerprints? && + location_fingerprint == other.location_fingerprint && + identifier_fingerprint == other.identifier_fingerprint + end + + def hash + location_fingerprint.hash ^ identifier_fingerprint.hash + end + + alias_method :eql?, :== + + protected + + attr_reader :location_fingerprint, :identifier_fingerprint + + def has_fingerprints? + location_fingerprint.present? && identifier_fingerprint.present? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/finding_signature.rb b/lib/gitlab/ci/reports/security/finding_signature.rb new file mode 100644 index 00000000000..d1d7ef5c377 --- /dev/null +++ b/lib/gitlab/ci/reports/security/finding_signature.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class FindingSignature + include VulnerabilityFindingSignatureHelpers + + attr_accessor :algorithm_type, :signature_value + + def initialize(params = {}) + @algorithm_type = params.dig(:algorithm_type) + @signature_value = params.dig(:signature_value) + end + + def signature_sha + Digest::SHA1.digest(signature_value) + end + + def signature_hex + signature_sha.unpack1("H*") + end + + def to_hash + { + algorithm_type: algorithm_type, + signature_sha: signature_sha + } + end + + def valid? + algorithm_types.key?(algorithm_type) + end + + def eql?(other) + other.algorithm_type == algorithm_type && + other.signature_sha == signature_sha + end + + alias_method :==, :eql? + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/base.rb b/lib/gitlab/ci/reports/security/locations/base.rb new file mode 100644 index 00000000000..9ad1d81287f --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/base.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Base + include ::Gitlab::Utils::StrongMemoize + + def ==(other) + other.fingerprint == fingerprint + end + + def fingerprint + strong_memoize(:fingerprint) do + Digest::SHA1.hexdigest(fingerprint_data) + end + end + + def as_json(options = nil) + fingerprint # side-effect call to initialize the ivar for serialization + + super + end + + def fingerprint_path + fingerprint_data + end + + private + + def fingerprint_data + raise NotImplementedError + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/sast.rb b/lib/gitlab/ci/reports/security/locations/sast.rb new file mode 100644 index 00000000000..23ffa91e720 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/sast.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class Sast < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/locations/secret_detection.rb b/lib/gitlab/ci/reports/security/locations/secret_detection.rb new file mode 100644 index 00000000000..0fd5cc5af11 --- /dev/null +++ b/lib/gitlab/ci/reports/security/locations/secret_detection.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + module Locations + class SecretDetection < Base + include Security::Concerns::FingerprintPathFromFile + + attr_reader :class_name + attr_reader :end_line + attr_reader :file_path + attr_reader :method_name + attr_reader :start_line + + def initialize(file_path:, start_line:, end_line: nil, class_name: nil, method_name: nil) + @class_name = class_name + @end_line = end_line + @file_path = file_path + @method_name = method_name + @start_line = start_line + end + + def fingerprint_data + "#{file_path}:#{start_line}:#{end_line}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb new file mode 100644 index 00000000000..1ba2d909d99 --- /dev/null +++ b/lib/gitlab/ci/reports/security/report.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Report + attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers + attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version + + delegate :project_id, to: :pipeline + + def initialize(type, pipeline, created_at) + @type = type + @pipeline = pipeline + @created_at = created_at + @findings = [] + @scanners = {} + @identifiers = {} + @scanned_resources = [] + @errors = [] + end + + def commit_sha + pipeline.sha + end + + def add_error(type, message = 'An unexpected error happened!') + errors << { type: type, message: message } + end + + def errored? + errors.present? + end + + def add_scanner(scanner) + scanners[scanner.key] ||= scanner + end + + def add_identifier(identifier) + identifiers[identifier.key] ||= identifier + end + + def add_finding(finding) + findings << finding + end + + def clone_as_blank + Report.new(type, pipeline, created_at) + end + + def replace_with!(other) + instance_variables.each do |ivar| + instance_variable_set(ivar, other.public_send(ivar.to_s[1..-1])) # rubocop:disable GitlabSecurity/PublicSend + end + end + + def merge!(other) + replace_with!(::Security::MergeReportsService.new(self, other).execute) + end + + def primary_scanner + scanners.first&.second + end + + def primary_scanner_order_to(other) + return 1 unless primary_scanner + return -1 unless other.primary_scanner + + primary_scanner <=> other.primary_scanner + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb new file mode 100644 index 00000000000..b7a5e36b108 --- /dev/null +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class Reports + attr_reader :reports, :pipeline + + delegate :each, :empty?, to: :reports + + def initialize(pipeline) + @reports = {} + @pipeline = pipeline + end + + def get_report(report_type, report_artifact) + reports[report_type] ||= Report.new(report_type, pipeline, report_artifact.created_at) + end + + def findings + reports.values.flat_map(&:findings) + end + + def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels) + unsafe_findings_count(target_reports, severity_levels) > vulnerabilities_allowed + end + + private + + def findings_diff(target_reports) + findings - target_reports&.findings.to_a + end + + def unsafe_findings_count(target_reports, severity_levels) + findings_diff(target_reports).count {|finding| finding.unsafe?(severity_levels)} + end + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb new file mode 100644 index 00000000000..6cb2e0ddb33 --- /dev/null +++ b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb @@ -0,0 +1,163 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Reports + module Security + class VulnerabilityReportsComparer + include Gitlab::Utils::StrongMemoize + + attr_reader :base_report, :head_report + + ACCEPTABLE_REPORT_AGE = 1.week + + def initialize(project, base_report, head_report) + @base_report = base_report + @head_report = head_report + + @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures) + + if @signatures_enabled + @added_findings = [] + @fixed_findings = [] + calculate_changes + end + end + + def base_report_created_at + @base_report.created_at + end + + def head_report_created_at + @head_report.created_at + end + + def base_report_out_of_date + return false unless @base_report.created_at + + ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at + end + + def added + strong_memoize(:added) do + if @signatures_enabled + @added_findings + else + head_report.findings - base_report.findings + end + end + end + + def fixed + strong_memoize(:fixed) do + if @signatures_enabled + @fixed_findings + else + base_report.findings - head_report.findings + end + end + end + + private + + def calculate_changes + # This is a deconstructed version of the eql? method on + # Ci::Reports::Security::Finding. It: + # + # * precomputes for the head_findings (using FindingMatcher): + # * sets of signature shas grouped by priority + # * mappings of signature shas to the head finding object + # + # These are then used when iterating the base findings to perform + # fast(er) prioritized, signature-based comparisons between each base finding + # and the head findings. + # + # Both the head_findings and base_findings arrays are iterated once + + base_findings = base_report.findings + head_findings = head_report.findings + + matcher = FindingMatcher.new(head_findings) + + base_findings.each do |base_finding| + matched_head_finding = matcher.find_and_remove_match!(base_finding) + + @fixed_findings << base_finding if matched_head_finding.nil? + end + + @added_findings = matcher.unmatched_head_findings.values + end + end + + class FindingMatcher + attr_reader :unmatched_head_findings, :head_findings + + include Gitlab::Utils::StrongMemoize + + def initialize(head_findings) + @head_findings = head_findings + @unmatched_head_findings = @head_findings.index_by(&:object_id) + end + + def find_and_remove_match!(base_finding) + matched_head_finding = find_matched_head_finding_for(base_finding) + + # no signatures matched, so check the normal uuids of the base and head findings + # for a match + matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil? + + @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil? + + matched_head_finding + end + + private + + def find_matched_head_finding_for(base_finding) + base_signature = sorted_signatures_for(base_finding).find do |signature| + # at this point a head_finding exists that has a signature with a + # matching priority, and a matching sha --> lookup the actual finding + # object from head_signatures_shas + head_signatures_shas[signature.signature_sha].eql?(base_finding) + end + + base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil + end + + def sorted_signatures_for(base_finding) + base_finding.signatures.select { |signature| head_finding_signature?(signature) } + .sort_by { |sig| -sig.priority } + end + + def head_finding_signature?(signature) + head_signatures_priorities[signature.priority].include?(signature.signature_sha) + end + + def head_signatures_priorities + strong_memoize(:head_signatures_priorities) do + signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new } + + head_findings.each_with_object(signatures_priorities) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.priority].add(signature.signature_sha) + end + end + end + end + + def head_signatures_shas + strong_memoize(:head_signatures_shas) do + head_findings.each_with_object({}) do |head_finding, memo| + head_finding.signatures.each do |signature| + memo[signature.signature_sha] = head_finding + end + # for the final uuid check when no signatures have matched + memo[head_finding.uuid] = head_finding + end + end + end + end + end + end + end +end |