Add latest changes from gitlab-org/gitlab@master

2 files changed, 56 insertions, 24 deletions

diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb
index 2d8a182e808..1bc9bd719f9 100644
--- a/lib/gitlab/ci/parsers/security/common.rb
+++ b/lib/gitlab/ci/parsers/security/common.rb
@@ -14,6 +14,7 @@ module Gitlab
           def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false)
             @json_data = json_data
             @report = report
+            @project = report.project
             @validate = validate
             @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled
           end
@@ -51,22 +52,27 @@ module Gitlab
             #
             # After 15.0 we will enforce schema validation by default
             # See: https://gitlab.com/groups/gitlab-org/-/epics/6968
-            schema_validation_passed = schema_validator.valid?
+            schema_validator.deprecation_warnings.each { |deprecation_warning| report.add_warning('Schema', deprecation_warning) }
 
             if validate
-              schema_validator.errors.each { |error| report.add_error('Schema', error) } unless schema_validation_passed
+              schema_validation_passed = schema_validator.valid?
+
+              # Validation warnings are errors
+              schema_validator.errors.each { |error| report.add_error('Schema', error) }
+              schema_validator.warnings.each { |warning| report.add_error('Schema', warning) }
 
               schema_validation_passed
             else
-              # We treat all schema validation errors as warnings
+              # Validation warnings are warnings
               schema_validator.errors.each { |error| report.add_warning('Schema', error) }
+              schema_validator.warnings.each { |warning| report.add_warning('Schema', warning) }
 
               true
             end
           end
 
           def schema_validator
-            @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version)
+            @schema_validator ||= ::Gitlab::Ci::Parsers::Security::Validators::SchemaValidator.new(report.type, report_data, report.version, project: @project)
           end
 
           def report_data
@@ -136,7 +142,7 @@ module Gitlab
                 metadata_version: report_version,
                 details: data['details'] || {},
                 signatures: signatures,
-                project_id: report.project_id,
+                project_id: @project.id,
                 vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled))
           end
 
@@ -279,7 +285,7 @@ module Gitlab
               report_type: report.type,
               primary_identifier_fingerprint: primary_identifier&.fingerprint,
               location_fingerprint: location_fingerprint,
-              project_id: report.project_id
+              project_id: @project.id
             }
 
             if uuid_v5_name_components.values.any?(&:nil?)
diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
index 88853ea4fb6..ffe9d506c7d 100644
--- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
+++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb
@@ -26,19 +26,19 @@ module Gitlab
                                    8.0.0-rc1 8.0.1-rc1 8.1.0-rc1 9.0.0-rc1].freeze
 
             # These come from https://app.periscopedata.com/app/gitlab/895813/Secure-Scan-metrics?widget=12248944&udv=1385516
-            KNOWN_VERSIONS_TO_DEPRECATE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze
+            KNOWN_VERSIONS_TO_REMOVE = %w[0.1 1.0 1.0.0 1.2 1.3 10.0.0 12.1.0 13.1.0 2.0 2.1 2.1.0 2.3 2.3.0 2.4 3.0 3.0.0 3.0.6 3.13.2 V2.7.0].freeze
 
-            VERSIONS_TO_DEPRECATE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_DEPRECATE).freeze
+            VERSIONS_TO_REMOVE_IN_15_0 = (PREVIOUS_RELEASES + KNOWN_VERSIONS_TO_REMOVE).freeze
 
             DEPRECATED_VERSIONS = {
-              cluster_image_scanning: VERSIONS_TO_DEPRECATE_IN_15_0,
-              container_scanning: VERSIONS_TO_DEPRECATE_IN_15_0,
-              coverage_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0,
-              dast: VERSIONS_TO_DEPRECATE_IN_15_0,
-              api_fuzzing: VERSIONS_TO_DEPRECATE_IN_15_0,
-              dependency_scanning: VERSIONS_TO_DEPRECATE_IN_15_0,
-              sast: VERSIONS_TO_DEPRECATE_IN_15_0,
-              secret_detection: VERSIONS_TO_DEPRECATE_IN_15_0
+              cluster_image_scanning: VERSIONS_TO_REMOVE_IN_15_0,
+              container_scanning: VERSIONS_TO_REMOVE_IN_15_0,
+              coverage_fuzzing: VERSIONS_TO_REMOVE_IN_15_0,
+              dast: VERSIONS_TO_REMOVE_IN_15_0,
+              api_fuzzing: VERSIONS_TO_REMOVE_IN_15_0,
+              dependency_scanning: VERSIONS_TO_REMOVE_IN_15_0,
+              sast: VERSIONS_TO_REMOVE_IN_15_0,
+              secret_detection: VERSIONS_TO_REMOVE_IN_15_0
             }.freeze
 
             class Schema
@@ -86,15 +86,18 @@ module Gitlab
               end
             end
 
-            def initialize(report_type, report_data, report_version = nil)
+            def initialize(report_type, report_data, report_version = nil, project: nil)
               @report_type = report_type&.to_sym
               @report_data = report_data
               @report_version = report_version
+              @project = project
               @errors = []
               @warnings = []
+              @deprecation_warnings = []
 
               populate_errors
               populate_warnings
+              populate_deprecation_warnings
             end
 
             def valid?
@@ -102,25 +105,46 @@ module Gitlab
             end
 
             def populate_errors
-              if Feature.enabled?(:enforce_security_report_validation)
-                @errors += schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+              schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+
+              log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty?
+
+              if Feature.enabled?(:enforce_security_report_validation, @project)
+                @errors += schema_validation_errors
               else
-                @warnings += schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) }
+                @warnings += schema_validation_errors
               end
             end
 
             def populate_warnings
-              add_deprecated_report_version_message if report_uses_deprecated_schema_version?
               add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version?
             end
 
+            def populate_deprecation_warnings
+              add_deprecated_report_version_message if report_uses_deprecated_schema_version?
+            end
+
             def add_deprecated_report_version_message
+              log_warnings(problem_type: 'using_deprecated_schema_version')
+
               message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}"
-              add_message_as(level: :warning, message: message)
+              add_message_as(level: :deprecation_warning, message: message)
+            end
+
+            def log_warnings(problem_type:)
+              Gitlab::AppLogger.info(
+                message: 'security report schema validation problem',
+                security_report_type: report_type,
+                security_report_version: report_version,
+                project_id: @project.id,
+                security_report_failure: problem_type
+              )
             end
 
             def add_unsupported_report_version_message
-              if Feature.enabled?(:enforce_security_report_validation)
+              log_warnings(problem_type: 'using_unsupported_schema_version')
+
+              if Feature.enabled?(:enforce_security_report_validation, @project)
                 handle_unsupported_report_version(treat_as: :error)
               else
                 handle_unsupported_report_version(treat_as: :warning)
@@ -152,6 +176,8 @@ module Gitlab
 
             def add_message_as(level:, message:)
               case level
+              when :deprecation_warning
+                @deprecation_warnings << message
               when :error
                 @errors << message
               when :warning
@@ -159,7 +185,7 @@ module Gitlab
               end
             end
 
-            attr_reader :errors, :warnings
+            attr_reader :errors, :warnings, :deprecation_warnings
 
             private