diff options
Diffstat (limited to 'lib/gitlab/ci/parsers')
5 files changed, 371 insertions, 42 deletions
diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb new file mode 100644 index 00000000000..deb20a2138c --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + class Cyclonedx + SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze + COMPONENT_ATTRIBUTES = %w[type name version].freeze + + def parse!(blob, sbom_report) + @report = sbom_report + @data = Gitlab::Json.parse(blob) + + return unless valid? + + parse_report + rescue JSON::ParserError => e + report.add_error("Report JSON is invalid: #{e}") + end + + private + + attr_reader :json_data, :report, :data + + def schema_validator + @schema_validator ||= Validators::CyclonedxSchemaValidator.new(data) + end + + def valid? + valid_schema? && supported_spec_version? + end + + def supported_spec_version? + return true if SUPPORTED_SPEC_VERSIONS.include?(data['specVersion']) + + report.add_error( + "Unsupported CycloneDX spec version. Must be one of: %{versions}" \ + % { versions: SUPPORTED_SPEC_VERSIONS.join(', ') } + ) + + false + end + + def valid_schema? + return true if schema_validator.valid? + + schema_validator.errors.each { |error| report.add_error(error) } + + false + end + + def parse_report + parse_metadata_properties + parse_components + end + + def parse_metadata_properties + properties = data.dig('metadata', 'properties') + source = CyclonedxProperties.parse_source(properties) + report.set_source(source) if source + end + + def parse_components + data['components']&.each do |component| + next unless supported_component_type?(component['type']) + + report.add_component(component.slice(*COMPONENT_ATTRIBUTES)) + end + end + + def supported_component_type?(type) + ::Enums::Sbom.component_types.include?(type.to_sym) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb new file mode 100644 index 00000000000..3dc73544208 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + # Parses GitLab CycloneDX metadata properties which are defined by the taxonomy at + # https://gitlab.com/gitlab-org/security-products/gitlab-cyclonedx-property-taxonomy + # + # This parser knows how to process schema version 1 and will not attempt to parse + # later versions. Each source type has it's own namespace in the property schema, + # and is also given its own parser. Properties are filtered by namespace, + # and then passed to each source parser for processing. + class CyclonedxProperties + SUPPORTED_SCHEMA_VERSION = '1' + GITLAB_PREFIX = 'gitlab:' + SOURCE_PARSERS = { + 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning + }.freeze + SUPPORTED_PROPERTIES = %w[ + meta:schema_version + dependency_scanning:category + dependency_scanning:input_file:path + dependency_scanning:source_file:path + dependency_scanning:package_manager:name + dependency_scanning:language:name + ].freeze + + def self.parse_source(...) + new(...).parse_source + end + + def initialize(properties) + @properties = properties + end + + def parse_source + return unless properties.present? + return unless supported_schema_version? + + source + end + + private + + attr_reader :properties + + def property_data + @property_data ||= properties + .each_with_object({}) { |property, data| parse_property(property, data) } + end + + def parse_property(property, data) + name = property['name'] + value = property['value'] + + # The specification permits the name or value to be absent. + return unless name.present? && value.present? + return unless name.start_with?(GITLAB_PREFIX) + + namespaced_name = name.delete_prefix(GITLAB_PREFIX) + + return unless SUPPORTED_PROPERTIES.include?(namespaced_name) + + parse_name_value_pair(namespaced_name, value, data) + end + + def parse_name_value_pair(name, value, data) + # Each namespace in the property name reflects a key in the hash. + # A property with the name `dependency_scanning:input_file:path` + # and the value `package-lock.json` should be transformed into + # this data: + # {"dependency_scanning": {"input_file": {"path": "package-lock.json"}}} + keys = name.split(':') + + # Remove last item from the keys and use it to create + # the initial object. + last = keys.pop + + # Work backwards. For each key, create a new hash wrapping the previous one. + # Using `dependency_scanning:input_file:path` as an example: + # + # 1. memo = { "path" => "package-lock.json" } (arguments given to reduce) + # 2. memo = { "input_file" => memo } + # 3. memo = { "dependency_scanning" => memo } + property = keys.reverse.reduce({ last => value }) do |memo, key| + { key => memo } + end + + data.deep_merge!(property) + end + + def schema_version + @schema_version ||= property_data.dig('meta', 'schema_version') + end + + def supported_schema_version? + schema_version == SUPPORTED_SCHEMA_VERSION + end + + def source + @source ||= property_data + .slice(*SOURCE_PARSERS.keys) + .lazy + .filter_map { |namespace, data| SOURCE_PARSERS[namespace].source(data) } + .first + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb new file mode 100644 index 00000000000..ad04b3257f9 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class DependencyScanning + REQUIRED_ATTRIBUTES = [ + %w[input_file path] + ].freeze + + def self.source(...) + new(...).source + end + + def initialize(data) + @data = data + end + + def source + return unless required_attributes_present? + + { + 'type' => :dependency_scanning, + 'data' => data, + 'fingerprint' => fingerprint + } + end + + private + + attr_reader :data + + def required_attributes_present? + REQUIRED_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + + def fingerprint + Digest::SHA256.hexdigest(data.to_json) + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb new file mode 100644 index 00000000000..9d56e001c2f --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/validators/cyclonedx_schema_validator.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Validators + class CyclonedxSchemaValidator + SCHEMA_PATH = Rails.root.join('app', 'validators', 'json_schemas', 'cyclonedx_report.json').freeze + + def initialize(report_data) + @report_data = report_data + end + + def valid? + errors.empty? + end + + def errors + @errors ||= pretty_errors + end + + private + + def raw_errors + JSONSchemer.schema(SCHEMA_PATH).validate(@report_data) + end + + def pretty_errors + raw_errors.map { |error| JSONSchemer::Errors.pretty(error) } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index ee7733a081d..c075ada725a 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -38,13 +38,14 @@ module Gitlab def initialize(report_type, report_version) @report_type = report_type.to_sym @report_version = report_version.to_s + @supported_versions = SUPPORTED_VERSIONS[@report_type] end delegate :validate, to: :schemer private - attr_reader :report_type, :report_version + attr_reader :report_type, :report_version, :supported_versions def schemer JSONSchemer.schema(pathname) @@ -60,10 +61,24 @@ module Gitlab report_declared_version = File.join(root_path, report_version, file_name) return report_declared_version if File.file?(report_declared_version) + if latest_vendored_patch_version + latest_vendored_patch_version_file = File.join(root_path, latest_vendored_patch_version, file_name) + return latest_vendored_patch_version_file if File.file?(latest_vendored_patch_version) + end + earliest_supported_version = SUPPORTED_VERSIONS[report_type].min File.join(root_path, earliest_supported_version, file_name) end + def latest_vendored_patch_version + ::Security::ReportSchemaVersionMatcher.new( + report_declared_version: report_version, + supported_versions: supported_versions + ).call + rescue ArgumentError + nil + end + def file_name report_type == :api_fuzzing ? "dast-report-format.json" : "#{report_type.to_s.dasherize}-report-format.json" end @@ -79,29 +94,85 @@ module Gitlab @warnings = [] @deprecation_warnings = [] - populate_errors - populate_warnings + populate_schema_version_errors + populate_validation_errors populate_deprecation_warnings end - def valid? - errors.empty? + def populate_schema_version_errors + add_schema_version_errors if add_schema_version_error? end - def populate_errors - schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + def add_schema_version_errors + if report_version.nil? + template = _("Report version not provided,"\ + " %{report_type} report type supports versions: %{supported_schema_versions}."\ + " GitLab will attempt to validate this report against the earliest supported versions of this report"\ + " type, to show all the errors but will not ingest the report") + message = format(template, report_type: report_type, supported_schema_versions: supported_schema_versions) + else + template = _("Version %{report_version} for report type %{report_type} is unsupported, supported versions"\ + " for this report type are: %{supported_schema_versions}."\ + " GitLab will attempt to validate this report against the earliest supported versions of this report"\ + " type, to show all the errors but will not ingest the report") + message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + end - log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + log_warnings(problem_type: 'using_unsupported_schema_version') + add_message_as(level: :error, message: message) + end + + def add_schema_version_error? + !report_uses_supported_schema_version? && + !report_uses_deprecated_schema_version? && + !report_uses_supported_major_and_minor_schema_version? + end + + def report_uses_deprecated_schema_version? + DEPRECATED_VERSIONS[report_type].include?(report_version) + end + + def report_uses_supported_schema_version? + SUPPORTED_VERSIONS[report_type].include?(report_version) + end - if Feature.enabled?(:enforce_security_report_validation, @project) - @errors += schema_validation_errors + def report_uses_supported_major_and_minor_schema_version? + if !find_latest_patch_version.nil? + add_supported_major_minor_behavior_warning + true else - @warnings += schema_validation_errors + false end end - def populate_warnings - add_unsupported_report_version_message if !report_uses_supported_schema_version? && !report_uses_deprecated_schema_version? + def find_latest_patch_version + ::Security::ReportSchemaVersionMatcher.new( + report_declared_version: report_version, + supported_versions: SUPPORTED_VERSIONS[report_type] + ).call + rescue ArgumentError + nil + end + + def add_supported_major_minor_behavior_warning + template = _("This report uses a supported MAJOR.MINOR schema version but the PATCH version doesn't match"\ + " any vendored schema version. Validation will be attempted against version"\ + " %{find_latest_patch_version}") + + message = format(template, find_latest_patch_version: find_latest_patch_version) + + add_message_as( + level: :warning, + message: message + ) + end + + def populate_validation_errors + schema_validation_errors = schema.validate(report_data).map { |error| JSONSchemer::Errors.pretty(error) } + + log_warnings(problem_type: 'schema_validation_fails') unless schema_validation_errors.empty? + + @errors += schema_validation_errors end def populate_deprecation_warnings @@ -111,10 +182,19 @@ module Gitlab def add_deprecated_report_version_message log_warnings(problem_type: 'using_deprecated_schema_version') - message = "Version #{report_version} for report type #{report_type} has been deprecated, supported versions for this report type are: #{supported_schema_versions}" + template = _("Version %{report_version} for report type %{report_type} has been deprecated,"\ + " supported versions for this report type are: %{supported_schema_versions}."\ + " GitLab will attempt to parse and ingest this report if valid.") + + message = format(template, report_version: report_version, report_type: report_type, supported_schema_versions: supported_schema_versions) + add_message_as(level: :deprecation_warning, message: message) end + def valid? + errors.empty? + end + def log_warnings(problem_type:) Gitlab::AppLogger.info( message: 'security report schema validation problem', @@ -127,34 +207,6 @@ module Gitlab ) end - def add_unsupported_report_version_message - log_warnings(problem_type: 'using_unsupported_schema_version') - - if Feature.enabled?(:enforce_security_report_validation, @project) - handle_unsupported_report_version(treat_as: :error) - else - handle_unsupported_report_version(treat_as: :warning) - end - end - - def report_uses_deprecated_schema_version? - DEPRECATED_VERSIONS[report_type].include?(report_version) - end - - def report_uses_supported_schema_version? - SUPPORTED_VERSIONS[report_type].include?(report_version) - end - - def handle_unsupported_report_version(treat_as:) - if report_version.nil? - message = "Report version not provided, #{report_type} report type supports versions: #{supported_schema_versions}" - else - message = "Version #{report_version} for report type #{report_type} is unsupported, supported versions for this report type are: #{supported_schema_versions}" - end - - add_message_as(level: treat_as, message: message) - end - def supported_schema_versions SUPPORTED_VERSIONS[report_type].join(", ") end |