Add latest changes from gitlab-org/gitlab@master

5 files changed, 100 insertions, 55 deletions

diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb
index 95da376b7c1..eadcbc4de43 100644
--- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection.rb
@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-require_relative 'secret_detection/version'
 require_relative 'secret_detection/status'
 require_relative 'secret_detection/finding'
 require_relative 'secret_detection/response'
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb
index 9bded2dbf97..e8d83178b09 100644
--- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/finding.rb
@@ -18,6 +18,16 @@ module Gitlab
         self.class == other.class && other.state == state
       end
 
+      def to_h
+        {
+          blob_id: blob_id,
+          status: status,
+          line_number: line_number,
+          type: type,
+          description: description
+        }
+      end
+
       protected
 
       def state
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb
index 83fc65a9b33..20d630d5dbb 100644
--- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/scan.rb
@@ -34,9 +34,9 @@ module Gitlab
       # in case the compilation fails.
       def initialize(logger: Logger.new($stdout), ruleset_path: RULESET_FILE_PATH)
         @logger = logger
-        @rules = parse_ruleset ruleset_path
-        @keywords = create_keywords @rules
-        @matcher = build_pattern_matcher @rules
+        @rules = parse_ruleset(ruleset_path)
+        @keywords = create_keywords(rules)
+        @pattern_matcher = build_pattern_matcher(rules)
       end
 
       # Runs Secret Detection scan on the list of given blobs. Both the total scan duration and
@@ -52,49 +52,51 @@ module Gitlab
       #     status: One of the SecretDetection::Status values
       #     results: [SecretDetection::Finding]
       # }
-      #
-      #
       def secrets_scan(blobs, timeout: DEFAULT_SCAN_TIMEOUT_SECS, blob_timeout: DEFAULT_BLOB_TIMEOUT_SECS)
         return SecretDetection::Response.new(SecretDetection::Status::INPUT_ERROR) unless validate_scan_input(blobs)
 
-        Timeout.timeout timeout do
+        Timeout.timeout(timeout) do
           matched_blobs = filter_by_keywords(blobs)
 
           next SecretDetection::Response.new(SecretDetection::Status::NOT_FOUND) if matched_blobs.empty?
 
           secrets = find_secrets_bulk(matched_blobs, blob_timeout)
 
-          scan_status = overall_scan_status secrets
+          scan_status = overall_scan_status(secrets)
 
           SecretDetection::Response.new(scan_status, secrets)
         end
       rescue Timeout::Error => e
-        @logger.error "Secret Detection operation timed out: #{e}"
+        logger.error "Secret detection operation timed out: #{e}"
+
         SecretDetection::Response.new(SecretDetection::Status::SCAN_TIMEOUT)
       end
 
       private
 
-      attr_reader :logger, :rules, :keywords, :matcher
+      attr_reader :logger, :rules, :keywords, :pattern_matcher
 
       # parses given ruleset file and returns the parsed rules
       def parse_ruleset(ruleset_file_path)
         rules_data = TomlRB.load_file(ruleset_file_path)
         rules_data['rules']
       rescue StandardError => e
-        logger.error "Failed to parse Secret Detection ruleset from '#{ruleset_file_path}' path: #{e}"
+        logger.error "Failed to parse secret detection ruleset from '#{ruleset_file_path}' path: #{e}"
+
         raise RulesetParseError
       end
 
       # builds RE2::Set pattern matcher for the given rules
       def build_pattern_matcher(rules)
         matcher = RE2::Set.new
+
         rules.each do |rule|
-          matcher.add(rule['regex'])
+          matcher.add(rule["regex"])
         end
 
         unless matcher.compile
-          logger.error "Failed to compile Secret Detection rulesets in RE::Set"
+          logger.error "Failed to compile secret detection rulesets in RE::Set"
+
           raise RulesetCompilationError
         end
 
@@ -104,8 +106,9 @@ module Gitlab
       # creates and returns the unique set of rule matching keywords
       def create_keywords(rules)
         secrets_keywords = []
+
         rules.each do |rule|
-          secrets_keywords << rule['keywords']
+          secrets_keywords << rule["keywords"]
         end
 
         secrets_keywords.flatten.compact.to_set
@@ -126,14 +129,16 @@ module Gitlab
       # finds secrets in the given list of blobs
       def find_secrets_bulk(blobs, blob_timeout)
         found_secrets = []
+
         blobs.each do |blob|
-          found_secrets << Timeout.timeout(blob_timeout) do
-            find_secrets(blob)
-          end
+          found_secrets << Timeout.timeout(blob_timeout) { find_secrets(blob) }
         rescue Timeout::Error => e
-          logger.error "Secret Detection scan timed out on the blob(id:#{blob.id}): #{e}"
-          found_secrets << SecretDetection::Finding.new(blob.id,
-            SecretDetection::Status::BLOB_TIMEOUT)
+          logger.error "Secret detection scan timed out on the blob(id:#{blob.id}): #{e}"
+
+          found_secrets << SecretDetection::Finding.new(
+            blob.id,
+            SecretDetection::Status::BLOB_TIMEOUT
+          )
         end
 
         found_secrets.flatten.freeze
@@ -147,20 +152,28 @@ module Gitlab
           # ignore the line scan if it is suffixed with '#gitleaks:allow'
           next if line.end_with?(GITLEAKS_KEYWORD_IGNORE)
 
-          patterns = matcher.match(line, :exception => false)
+          patterns = pattern_matcher.match(line, :exception => false)
           next unless patterns.any?
 
-          line_no = index + 1
+          line_number = index + 1
           patterns.each do |pattern|
-            type = rules[pattern]['id']
-            description = rules[pattern]['description']
-            secrets << SecretDetection::Finding.new(blob.id, SecretDetection::Status::FOUND, line_no, type,
-              description)
+            type = rules[pattern]["id"]
+            description = rules[pattern]["description"]
+
+            secrets << SecretDetection::Finding.new(
+              blob.id,
+              SecretDetection::Status::FOUND,
+              line_number,
+              type,
+              description
+            )
           end
         end
+
         secrets
       rescue StandardError => e
-        logger.error "Secret Detection scan failed on the blob(id:#{blob.id}): #{e}"
+        logger.error "Secret detection scan failed on the blob(id:#{blob.id}): #{e}"
+
         SecretDetection::Finding.new(blob.id, SecretDetection::Status::SCAN_ERROR)
       end
 
diff --git a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb
index 45ac04a81b7..200294fc2e7 100644
--- a/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb
+++ b/gems/gitlab-secret_detection/lib/gitlab/secret_detection/status.rb
@@ -2,7 +2,7 @@
 
 module Gitlab
   module SecretDetection
-    # All the possible statuses emitted by the Scan operation
+    # All the possible statuses emitted by the scan operation
     class Status
       NOT_FOUND = 0 # When scan operation completes with zero findings
       FOUND = 1 # When scan operation completes with one or more findings
diff --git a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb
index dfe3fdf4bb9..cd268099dd0 100644
--- a/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb
+++ b/gems/gitlab-secret_detection/spec/lib/gitlab/secret_detection/scan_spec.rb
@@ -13,26 +13,34 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
     {
       "title" => "gitleaks config",
       "rules" => [
-        { "id" => "gitlab_personal_access_token",
+        {
+          "id" => "gitlab_personal_access_token",
           "description" => "GitLab Personal Access Token",
           "regex" => "glpat-[0-9a-zA-Z_\\-]{20}",
           "tags" => %w[gitlab revocation_type],
-          "keywords" => ["glpat"] },
-        { "id" => "gitlab_pipeline_trigger_token",
+          "keywords" => ["glpat"]
+        },
+        {
+          "id" => "gitlab_pipeline_trigger_token",
           "description" => "GitLab Pipeline Trigger Token",
           "regex" => "glptt-[0-9a-zA-Z_\\-]{20}",
           "tags" => ["gitlab"],
-          "keywords" => ["glptt"] },
-        { "id" => "gitlab_runner_registration_token",
+          "keywords" => ["glptt"]
+        },
+        {
+          "id" => "gitlab_runner_registration_token",
           "description" => "GitLab Runner Registration Token",
           "regex" => "GR1348941[0-9a-zA-Z_-]{20}",
           "tags" => ["gitlab"],
-          "keywords" => ["GR1348941"] },
-        { "id" => "gitlab_feed_token",
+          "keywords" => ["GR1348941"]
+        },
+        {
+          "id" => "gitlab_feed_token",
           "description" => "GitLab Feed Token",
           "regex" => "glft-[0-9a-zA-Z_-]{20}",
           "tags" => ["gitlab"],
-          "keywords" => ["glft"] }
+          "keywords" => ["glft"]
+        }
       ]
     }
   end
@@ -65,16 +73,21 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
 
       it "does not match" do
         expected_response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::NOT_FOUND)
+
         expect(scan.secrets_scan(blobs)).to eq(expected_response)
       end
 
       it "attempts to keyword match returning no blobs for further scan" do
-        expect(scan).to receive(:filter_by_keywords).with(blobs).and_return([])
+        expect(scan).to receive(:filter_by_keywords)
+          .with(blobs)
+          .and_return([])
+
         scan.secrets_scan(blobs)
       end
 
       it "does not attempt to regex match" do
         expect(scan).not_to receive(:match_rules_bulk)
+
         scan.secrets_scan(blobs)
       end
     end
@@ -89,7 +102,7 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
         ]
       end
 
-      it "matches glpat" do
+      it "matches different types of rules" do
         expected_response = Gitlab::SecretDetection::Response.new(
           Gitlab::SecretDetection::Status::FOUND,
           [
@@ -129,6 +142,8 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
     end
 
     context "when configured with time out" do
+      let(:each_blob_timeout_secs) { 0.000_001 } # 1 micro-sec to intentionally timeout large blob
+
       let(:large_data) do
         ("large data with a secret glpat-12312312312312312312\n" * 10_000_000).freeze # gitleaks:allow
       end
@@ -141,40 +156,44 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
         ]
       end
 
+      let(:all_large_blobs) do
+        [
+          new_blob(id: 111, data: large_data),
+          new_blob(id: 222, data: large_data),
+          new_blob(id: 333, data: large_data)
+        ]
+      end
+
       it "whole secret detection scan operation times out" do
         scan_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob
+
         response = Gitlab::SecretDetection::Response.new(Gitlab::SecretDetection::Status::SCAN_TIMEOUT)
+
         expect(scan.secrets_scan(blobs, timeout: scan_timeout_secs)).to eq(response)
       end
 
       it "one of the blobs times out while others continue to get scanned" do
-        each_blob_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob
-
         expected_response = Gitlab::SecretDetection::Response.new(
           Gitlab::SecretDetection::Status::FOUND_WITH_ERRORS,
           [
             Gitlab::SecretDetection::Finding.new(
-              blobs[0].id, Gitlab::SecretDetection::Status::FOUND, 1,
+              blobs[0].id,
+              Gitlab::SecretDetection::Status::FOUND,
+              1,
               ruleset['rules'][2]['id'],
               ruleset['rules'][2]['description']
             ),
             Gitlab::SecretDetection::Finding.new(
-              blobs[2].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+              blobs[2].id,
+              Gitlab::SecretDetection::Status::BLOB_TIMEOUT
             )
-          ])
+          ]
+        )
 
         expect(scan.secrets_scan(blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response)
       end
 
       it "all the blobs time out" do
-        each_blob_timeout_secs = 0.000_001 # 1 micro-sec to intentionally timeout large blob
-
-        all_large_blobs = [
-          new_blob(id: 111, data: large_data),
-          new_blob(id: 222, data: large_data),
-          new_blob(id: 333, data: large_data)
-        ]
-
         # scan status changes to SCAN_TIMEOUT when *all* the blobs time out
         expected_scan_status = Gitlab::SecretDetection::Status::SCAN_TIMEOUT
 
@@ -182,15 +201,19 @@ RSpec.describe Gitlab::SecretDetection::Scan, feature_category: :secret_detectio
           expected_scan_status,
           [
             Gitlab::SecretDetection::Finding.new(
-              all_large_blobs[0].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+              all_large_blobs[0].id,
+              Gitlab::SecretDetection::Status::BLOB_TIMEOUT
             ),
             Gitlab::SecretDetection::Finding.new(
-              all_large_blobs[1].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+              all_large_blobs[1].id,
+              Gitlab::SecretDetection::Status::BLOB_TIMEOUT
             ),
             Gitlab::SecretDetection::Finding.new(
-              all_large_blobs[2].id, Gitlab::SecretDetection::Status::BLOB_TIMEOUT
+              all_large_blobs[2].id,
+              Gitlab::SecretDetection::Status::BLOB_TIMEOUT
             )
-          ])
+          ]
+        )
 
         expect(scan.secrets_scan(all_large_blobs, blob_timeout: each_blob_timeout_secs)).to eq(expected_response)
       end