diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-12-12 18:13:14 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-12-12 18:13:14 +0300 |
commit | 86a3b1b3ae2115c6ab7b9d492e9c89ac70963d3a (patch) | |
tree | c78266705382f28d08786303ea57525ec1e2a2cf /gems | |
parent | 7045d3816aa7bab6619e9d841f45d5cb8d454f23 (diff) |
Add latest changes from gitlab-org/gitlab@master
Diffstat (limited to 'gems')
5 files changed, 100 insertions, 55 deletions
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb index 95da376b7c1..eadcbc4de43 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -require_relative 'secret_detection/version' require_relative 'secret_detection/status' require_relative 'secret_detection/finding' require_relative 'secret_detection/response' diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb index 9bded2dbf97..e8d83178b09 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb @@ -18,6 +18,16 @@ module Gitlab self.class == other.class && other.state == state end + def to_h + { + blob_id: blob_id, + status: status, + line_number: line_number, + type: type, + description: description + } + end + protected def state diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb index 83fc65a9b33..20d630d5dbb 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb @@ -34,9 +34,9 @@ module Gitlab # in case the compilation fails. def initialize(logger: Logger.new($stdout), ruleset_path: RULESET_FILE_PATH) @logger = logger - @rules = parse_ruleset ruleset_path - @keywords = create_keywords @rules - @matcher = build_pattern_matcher @rules + @rules = parse_ruleset(ruleset_path) + @keywords = create_keywords(rules) + @pattern_matcher = build_pattern_matcher(rules) end # Runs Secret Detection scan on the list of given blobs. Both the total scan duration and @@ -52,49 +52,51 @@ module Gitlab # status: One of the SecretDetection::Status values # results: [SecretDetection::Finding] # } - # - # def secrets_scan(blobs, timeout: DEFAULT_SCAN_TIMEOUT_SECS, blob_timeout: DEFAULT_BLOB_TIMEOUT_SECS) return SecretDetection::Response.new(SecretDetection::Status::INPUT_ERROR) unless validate_scan_input(blobs) - Timeout.timeout timeout do + Timeout.timeout(timeout) do matched_blobs = filter_by_keywords(blobs) next SecretDetection::Response.new(SecretDetection::Status::NOT_FOUND) if matched_blobs.empty? secrets = find_secrets_bulk(matched_blobs, blob_timeout) - scan_status = overall_scan_status secrets + scan_status = overall_scan_status(secrets) SecretDetection::Response.new(scan_status, secrets) end rescue Timeout::Error => e - @logger.error "Secret Detection operation timed out: #{e}" + logger.error "Secret detection operation timed out: #{e}" + SecretDetection::Response.new(SecretDetection::Status::SCAN_TIMEOUT) end private - attr_reader :logger, :rules, :keywords, :matcher + attr_reader :logger, :rules, :keywords, :pattern_matcher # parses given ruleset file and returns the parsed rules def parse_ruleset(ruleset_file_path) rules_data = TomlRB.load_file(ruleset_file_path) rules_data['rules'] rescue StandardError => e - logger.error "Failed to parse Secret Detection ruleset from '#{ruleset_file_path}' path: #{e}" + logger.error "Failed to parse secret detection ruleset from '#{ruleset_file_path}' path: #{e}" + raise RulesetParseError end # builds RE2::Set pattern matcher for the given rules def build_pattern_matcher(rules) matcher = RE2::Set.new + rules.each do |rule| - matcher.add(rule['regex']) + matcher.add(rule["regex"]) end unless matcher.compile - logger.error "Failed to compile Secret Detection rulesets in RE::Set" + logger.error "Failed to compile secret detection rulesets in RE::Set" + raise RulesetCompilationError end @@ -104,8 +106,9 @@ module Gitlab # creates and returns the unique set of rule matching keywords def create_keywords(rules) secrets_keywords = [] + rules.each do |rule| - secrets_keywords << rule['keywords'] + secrets_keywords << rule["keywords"] end secrets_keywords.flatten.compact.to_set @@ -126,14 +129,16 @@ module Gitlab # finds secrets in the given list of blobs def find_secrets_bulk(blobs, blob_timeout) found_secrets = [] + blobs.each do |blob| - found_secrets << Timeout.timeout(blob_timeout) do - find_secrets(blob) - end + found_secrets << Timeout.timeout(blob_timeout) { find_secrets(blob) } rescue Timeout::Error => e - logger.error "Secret Detection scan timed out on the blob(id:#{blob.id}): #{e}" - found_secrets << SecretDetection::Finding.new(blob.id, - SecretDetection::Status::BLOB_TIMEOUT) + logger.error "Secret detection scan timed out on the blob(id:#{blob.id}): #{e}" + + found_secrets << SecretDetection::Finding.new( + blob.id, + SecretDetection::Status::BLOB_TIMEOUT + ) end found_secrets.flatten.freeze @@ -147,20 +152,28 @@ module Gitlab # ignore the line scan if it is suffixed with '#gitleaks:allow' next if line.end_with?(GITLEAKS_KEYWORD_IGNORE) - patterns = matcher.match(line, :exception => false) + patterns = pattern_matcher.match(line, :exception => false) next unless patterns.any? - line_no = index + 1 + line_number = index + 1 patterns.each do |pattern| - type = rules[pattern]['id'] - description = rules[pattern]['description'] - secrets << SecretDetection::Finding.new(blob.id, SecretDetection::Status::FOUND, line_no, type, - description) + type = rules[pattern]["id"] + description = rules[pattern]["description"] + + secrets << SecretDetection::Finding.new( + blob.id, + SecretDetection::Status::FOUND, + line_number, + type, + description + ) end end + secrets rescue StandardError => e - logger.error "Secret Detection scan failed on the blob(id:#{blob.id}): #{e}" + logger.error "Secret detection scan failed on the blob(id:#{blob.id}): #{e}" + SecretDetection::Finding.new(blob.id, SecretDetection::Status::SCAN_ERROR) end diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb index 45ac04a81b7..200294fc2e7 100644 --- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb +++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb @@ -2,7 +2,7 @@ module Gitlab module SecretDetection - # All the possible statuses emitted by the Scan operation + # All the possible statuses emitted by the scan operation class Status NOT_FOUND = 0 # When scan operation completes with zero findings FOUND = 1 # When scan operation completes with one or more findings diff --git a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb index dfe3fdf4bb9..cd268099dd0 100644 --- a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb +++ b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb @@ -13,26 +13,34 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio { "title" => "gitleaks config", "rules" => [ - { "id" => "gitlab_personal_access_token", + { + "id" => "gitlab_personal_access_token", "description" => "GitLab Personal Access Token", "regex" => "glpat-[0-9a-zA-Z_\\-]{20}", "tags" => %w[gitlab revocation_type], - "keywords" => ["glpat"] }, - { "id" => "gitlab_pipeline_trigger_token", + "keywords" => ["glpat"] + }, + { + "id" => "gitlab_pipeline_trigger_token", "description" => "GitLab Pipeline Trigger Token", "regex" => "glptt-[0-9a-zA-Z_\\-]{20}", "tags" => ["gitlab"], - "keywords" => ["glptt"] }, - { "id" => "gitlab_runner_registration_token", + "keywords" => ["glptt"] + }, + { + "id" => "gitlab_runner_registration_token", "description" => "GitLab Runner Registration Token", "regex" => "GR1348941[0-9a-zA-Z_-]{20}", "tags" => ["gitlab"], - "keywords" => ["GR1348941"] }, - { "id" => "gitlab_feed_token", + "keywords" => ["GR1348941"] + }, + { + "id" => "gitlab_feed_token", "description" => "GitLab Feed Token", "regex" => "glft-[0-9a-zA-Z_-]{20}", "tags" => ["gitlab"], - "keywords" => ["glft"] } + "keywords" => ["glft"] + } ] } end @@ -65,16 +73,21 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio it "does not match" do expected_response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND) + expect(scan.secrets_scan(blobs)).to eq(expected_response) end it "attempts to keyword match returning no blobs for further scan" do - expect(scan).to receive(:filter_by_keywords).with(blobs).and_return([]) + expect(scan).to receive(:filter_by_keywords) + .with(blobs) + .and_return([]) + scan.secrets_scan(blobs) end it "does not attempt to regex match" do expect(scan).not_to receive(:match_rules_bulk) + scan.secrets_scan(blobs) end end @@ -89,7 +102,7 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio ] end - it "matches glpat" do + it "matches different types of rules" do expected_response = Gitlab::SecretDetection::Response.new( Gitlab::SecretDetection::Status::FOUND, [ @@ -129,6 +142,8 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio end context "when configured with time out" do + let(:each_blob_timeout_secs) { 0.000_001 } # 1 micro-sec to intentionally timeout large blob + let(:large_data) do ("large data with a secret glpat-12312312312312312312\n" * 10_000_000).freeze # gitleaks:allow end @@ -141,40 +156,44 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio ] end + let(:all_large_blobs) do + [ + new_blob(id: 111, data: large_data), + new_blob(id: 222, data: large_data), + new_blob(id: 333, data: large_data) + ] + end + it "whole secret detection scan operation times out" do scan_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob + response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::SCAN_TIMEOUT) + expect(scan.secrets_scan(blobs, timeout: scan_timeout_secs)).to eq(response) end it "one of the blobs times out while others continue to get scanned" do - each_blob_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob - expected_response = Gitlab::SecretDetection::Response.new( Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS, [ Gitlab::SecretDetection::Finding.new( - blobs[0].id, Gitlab::SecretDetection::Status::FOUND, 1, + blobs[0].id, + Gitlab::SecretDetection::Status::FOUND, + 1, ruleset['rules'][2]['id'], ruleset['rules'][2]['description'] ), Gitlab::SecretDetection::Finding.new( - blobs[2].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT + blobs[2].id, + Gitlab::SecretDetection::Status::BLOB_TIMEOUT ) - ]) + ] + ) expect(scan.secrets_scan(blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response) end it "all the blobs time out" do - each_blob_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob - - all_large_blobs = [ - new_blob(id: 111, data: large_data), - new_blob(id: 222, data: large_data), - new_blob(id: 333, data: large_data) - ] - # scan status changes to SCAN_TIMEOUT when *all* the blobs time out expected_scan_status = Gitlab::SecretDetection::Status::SCAN_TIMEOUT @@ -182,15 +201,19 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio expected_scan_status, [ Gitlab::SecretDetection::Finding.new( - all_large_blobs[0].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_blobs[0].id, + Gitlab::SecretDetection::Status::BLOB_TIMEOUT ), Gitlab::SecretDetection::Finding.new( - all_large_blobs[1].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_blobs[1].id, + Gitlab::SecretDetection::Status::BLOB_TIMEOUT ), Gitlab::SecretDetection::Finding.new( - all_large_blobs[2].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT + all_large_blobs[2].id, + Gitlab::SecretDetection::Status::BLOB_TIMEOUT ) - ]) + ] + ) expect(scan.secrets_scan(all_large_blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response) end |