diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-05-17 19:05:49 +0300 |
| commit | 43a25d93ebdabea52f99b05e15b06250cd8f07d7 (patch) | |
| tree | dceebdc68925362117480a5d672bcff122fb625b /lib/gitlab | |
| parent | 20c84b99005abd1c82101dfeff264ac50d2df211 (diff) | |
Add latest changes from gitlab-org/gitlab@16-0-stable-eev16.0.0-rc42
Diffstat (limited to 'lib/gitlab')
722 files changed, 11233 insertions, 71525 deletions
diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index fa025a2658f..bafda11170a 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -16,6 +16,7 @@ module Gitlab DEVELOPER = 30 MAINTAINER = 40 OWNER = 50 + ADMIN = 60 # Branch protection settings PROTECTION_NONE = 0 diff --git a/lib/gitlab/action_cable/request_store_callbacks.rb b/lib/gitlab/action_cable/request_store_callbacks.rb index a9f30b0fc10..14d80a7c40c 100644 --- a/lib/gitlab/action_cable/request_store_callbacks.rb +++ b/lib/gitlab/action_cable/request_store_callbacks.rb @@ -5,8 +5,6 @@ module Gitlab module RequestStoreCallbacks def self.install ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper - ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper - ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper end def self.wrapper diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb index 07dc4c02ba8..2143497f084 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/records_fetcher.rb @@ -94,10 +94,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_associations(records) - ActiveRecord::Associations::Preloader.new.preload( - records, - MAPPINGS.fetch(subject_class).fetch(:includes_for_query) - ) + ActiveRecord::Associations::Preloader.new( + records: records, + associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ).call records end diff --git a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb index b00925495f2..1d041e76277 100644 --- a/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/aggregated/stage_query_helpers.rb @@ -32,7 +32,7 @@ module Gitlab end def duration_in_seconds(duration_expression = duration) - Arel::Nodes::Extract.new(duration_expression, :epoch) + Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(duration_expression, :epoch).as('double precision')]) end end end diff --git a/lib/gitlab/analytics/cycle_analytics/average.rb b/lib/gitlab/analytics/cycle_analytics/average.rb index 7140d31d536..4113e2e5d6a 100644 --- a/lib/gitlab/analytics/cycle_analytics/average.rb +++ b/lib/gitlab/analytics/cycle_analytics/average.rb @@ -41,7 +41,7 @@ module Gitlab end def average_in_seconds - Arel::Nodes::Extract.new(average, :epoch) + Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(average, :epoch).as('double precision')]) end end end diff --git a/lib/gitlab/analytics/cycle_analytics/median.rb b/lib/gitlab/analytics/cycle_analytics/median.rb index 5775d0324c6..0958cc39945 100644 --- a/lib/gitlab/analytics/cycle_analytics/median.rb +++ b/lib/gitlab/analytics/cycle_analytics/median.rb @@ -38,7 +38,8 @@ module Gitlab end def median_duration_in_seconds - Arel::Nodes::Extract.new(percentile_cont, :epoch) + Arel::Nodes::NamedFunction.new('CAST', + [Arel::Nodes::Extract.new(percentile_cont, :epoch).as('double precision')]) end end end diff --git a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb index 140c4a300ca..9deb5072112 100644 --- a/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb +++ b/lib/gitlab/analytics/cycle_analytics/records_fetcher.rb @@ -67,10 +67,10 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_associations(records) # using preloader instead of includes to avoid AR generating a large column list - ActiveRecord::Associations::Preloader.new.preload( - records, - MAPPINGS.fetch(subject_class).fetch(:includes_for_query) - ) + ActiveRecord::Associations::Preloader.new( + records: records, + associations: MAPPINGS.fetch(subject_class).fetch(:includes_for_query) + ).call records end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index 2df3680db5f..2c4b0215307 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -38,13 +38,12 @@ module Gitlab attribute :created_after, :datetime attribute :created_before, :datetime - attribute :group + attribute :namespace attribute :current_user attribute :value_stream attribute :sort attribute :direction attribute :page - attribute :project attribute :stage_id attribute :end_event_filter @@ -66,10 +65,6 @@ module Gitlab self.end_event_filter ||= Gitlab::Analytics::CycleAnalytics::BaseQueryBuilder::DEFAULT_END_EVENT_FILTER end - def project_ids - Array(@project_ids) - end - def to_data_collector_params { current_user: current_user, @@ -86,12 +81,9 @@ module Gitlab def to_data_attributes {}.tap do |attrs| - attrs[:aggregation] = aggregation_attributes if group - attrs[:group] = group_data_attributes if group attrs[:value_stream] = value_stream_data_attributes.to_json if value_stream attrs[:created_after] = created_after.to_date.iso8601 attrs[:created_before] = created_before.to_date.iso8601 - attrs[:projects] = group_projects(project_ids) if group && project_ids.present? attrs[:labels] = label_name.to_json if label_name.present? attrs[:assignees] = assignee_username.to_json if assignee_username.present? attrs[:author] = author_username if author_username.present? @@ -99,35 +91,64 @@ module Gitlab attrs[:sort] = sort if sort.present? attrs[:direction] = direction if direction.present? attrs[:stage] = stage_data_attributes.to_json if stage_id.present? + attrs[:namespace] = namespace_attributes + attrs[:enable_tasks_by_type_chart] = 'false' + attrs[:enable_customizable_stages] = 'false' + attrs[:enable_projects_filter] = 'false' + attrs[:default_stages] = Gitlab::Analytics::CycleAnalytics::DefaultStages.all.map do |stage_params| + ::Analytics::CycleAnalytics::StagePresenter.new(stage_params) + end.to_json + + attrs.merge!(foss_project_level_params, resource_paths) end end + def project_ids + Array(@project_ids) + end + private - def use_aggregated_backend? - # for now it's only available on the group-level - group.present? - end + delegate :url_helpers, to: Gitlab::Routing + + def foss_project_level_params + return {} unless project - def aggregation_attributes { - enabled: aggregation.enabled.to_s, - last_run_at: aggregation.last_incremental_run_at&.iso8601, - next_run_at: aggregation.estimated_next_run_at&.iso8601 + project_id: project.id, + group_path: project.group ? "groups/#{project.group&.full_path}" : nil, + request_path: url_helpers.project_cycle_analytics_path(project), + full_path: project.full_path } end - def aggregation - @aggregation ||= ::Analytics::CycleAnalytics::Aggregation.safe_create_for_namespace(group) + def resource_paths + helpers = ActionController::Base.helpers + + {}.tap do |paths| + paths[:empty_state_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:no_data_svg_path] = helpers.image_path("illustrations/analytics/cycle-analytics-empty-chart.svg") + paths[:no_access_svg_path] = helpers.image_path("illustrations/analytics/no-access.svg") + + if project + paths[:milestones_path] = url_helpers.project_milestones_path(project, format: :json) + paths[:labels_path] = url_helpers.project_labels_path(project, format: :json) + end + end end - def group_data_attributes + # FOSS version doesn't use the aggregated VSA backend + def use_aggregated_backend? + false + end + + def namespace_attributes + return {} unless project + { - id: group.id, - namespace_id: group.id, - name: group.name, - full_path: group.full_path, - avatar_url: group.avatar_url + name: project.name, + full_path: project.full_path, + type: namespace.type } end @@ -139,28 +160,6 @@ module Gitlab } end - def group_projects(project_ids) - GroupProjectsFinder.new( - group: group, - current_user: current_user, - options: { include_subgroups: true }, - project_ids_relation: project_ids - ) - .execute - .with_route - .map { |project| project_data_attributes(project) } - .to_json - end - - def project_data_attributes(project) - { - id: project.to_gid.to_s, - name: project.name, - path_with_namespace: project.path_with_namespace, - avatar_url: project.avatar_url - } - end - def stage_data_attributes return unless stage @@ -196,10 +195,18 @@ module Gitlab return unless value_stream strong_memoize(:stage) do - ::Analytics::CycleAnalytics::StageFinder.new(parent: project&.project_namespace || group, stage_id: stage_id).execute if stage_id + ::Analytics::CycleAnalytics::StageFinder.new(parent: namespace, stage_id: stage_id).execute if stage_id + end + end + + def project + strong_memoize(:project) do + namespace.project if namespace.is_a?(Namespaces::ProjectNamespace) end end end end end end + +Gitlab::Analytics::CycleAnalytics::RequestParams.prepend_mod_with('Gitlab::Analytics::CycleAnalytics::RequestParams') diff --git a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb index 9b4cbc9090c..85dcc773e2b 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_events/plan_stage_start.rb @@ -6,7 +6,7 @@ module Gitlab module StageEvents class PlanStageStart < MetricsBasedStageEvent def self.name - s_("CycleAnalyticsEvent|Issue first associated with a milestone or issue first added to a board") + s_("CycleAnalyticsEvent|Issue first associated with a milestone or first added to a board") end def self.identifier diff --git a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb index 5648984ecbb..3416a916e26 100644 --- a/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb +++ b/lib/gitlab/analytics/cycle_analytics/stage_query_helpers.rb @@ -13,7 +13,9 @@ module Gitlab end def round_duration_to_seconds - Arel::Nodes::NamedFunction.new('ROUND', [Arel::Nodes::Extract.new(duration, :epoch)]) + Arel::Nodes::NamedFunction.new('ROUND', [ + Arel::Nodes::NamedFunction.new('CAST', [Arel::Nodes::Extract.new(duration, :epoch).as('double precision')]) + ]) end def duration diff --git a/lib/gitlab/app_logger.rb b/lib/gitlab/app_logger.rb index a39e7f31886..decc6be3410 100644 --- a/lib/gitlab/app_logger.rb +++ b/lib/gitlab/app_logger.rb @@ -2,18 +2,12 @@ module Gitlab class AppLogger < Gitlab::MultiDestinationLogger - LOGGERS = [Gitlab::AppTextLogger, Gitlab::AppJsonLogger].freeze - def self.loggers - if Gitlab::Utils.to_boolean(ENV.fetch('UNSTRUCTURED_RAILS_LOG', 'true')) - LOGGERS - else - [Gitlab::AppJsonLogger] - end + [Gitlab::AppJsonLogger] end def self.primary_logger - Gitlab::AppTextLogger + Gitlab::AppJsonLogger end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 06ce1dbdc77..0ea52b7b7c8 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -25,7 +25,8 @@ module Gitlab :artifact_used_cdn, :artifacts_dependencies_size, :artifacts_dependencies_count, - :root_caller_id + :root_caller_id, + :merge_action_status ].freeze private_constant :KNOWN_KEYS @@ -43,7 +44,8 @@ module Gitlab Attribute.new(:artifact_used_cdn, Object), Attribute.new(:artifacts_dependencies_size, Integer), Attribute.new(:artifacts_dependencies_count, Integer), - Attribute.new(:root_caller_id, String) + Attribute.new(:root_caller_id, String), + Attribute.new(:merge_action_status, String) ].freeze def self.known_keys @@ -97,6 +99,7 @@ module Gitlab assign_hash_if_value(hash, :artifact_used_cdn) assign_hash_if_value(hash, :artifacts_dependencies_size) assign_hash_if_value(hash, :artifacts_dependencies_count) + assign_hash_if_value(hash, :merge_action_status) hash[:user] = -> { username } if include_user? hash[:user_id] = -> { user_id } if include_user? diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index 466538df56e..a8e74cbd7e6 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -30,6 +30,7 @@ module Gitlab group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, group_testing_hook: { threshold: 5, interval: 1.minute }, + member_delete: { threshold: 60, interval: 1.minute }, profile_add_new_email: { threshold: 5, interval: 1.minute }, web_hook_calls: { interval: 1.minute }, web_hook_calls_mid: { interval: 1.minute }, @@ -55,8 +56,13 @@ module Gitlab phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, + project_fork_sync: { threshold: 10, interval: 30.minutes }, + ai_action: { threshold: 160, interval: 8.hours }, jobs_index: { threshold: 600, interval: 1.minute }, - bulk_import: { threshold: 6, interval: 1.minute } + bulk_import: { threshold: 6, interval: 1.minute }, + projects_api_rate_limit_unauthenticated: { + threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes + } }.freeze end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index fddc1f830aa..e3d2b394404 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -5,6 +5,10 @@ module Gitlab class Auditor attr_reader :scope, :name + PERMITTED_TARGET_CLASSES = [ + ::Operations::FeatureFlag + ].freeze + # Record audit events # # @param [Hash] context @@ -113,7 +117,11 @@ module Gitlab end def audit_enabled? - authentication_event? + authentication_event? || permitted_target? + end + + def permitted_target? + @target.class.in? PERMITTED_TARGET_CLASSES end def authentication_event? diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 06bdb2c1ddc..9268fdd8519 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -3,7 +3,7 @@ module Gitlab module Auth MissingPersonalAccessTokenError = Class.new(StandardError) - IpBlacklisted = Class.new(StandardError) + IpBlocked = Class.new(StandardError) # Scopes used for GitLab API access API_SCOPE = :api @@ -29,6 +29,12 @@ module Gitlab WRITE_REGISTRY_SCOPE = :write_registry REGISTRY_SCOPES = [READ_REGISTRY_SCOPE, WRITE_REGISTRY_SCOPE].freeze + # Scopes used for GitLab Observability access which is outside of the GitLab app itself. + # Hence the lack of ability mapping in `abilities_for_scopes`. + READ_OBSERVABILITY_SCOPE = :read_observability + WRITE_OBSERVABILITY_SCOPE = :write_observability + OBSERVABILITY_SCOPES = [READ_OBSERVABILITY_SCOPE, WRITE_OBSERVABILITY_SCOPE].freeze + # Scopes used for GitLab as admin SUDO_SCOPE = :sudo ADMIN_MODE_SCOPE = :admin_mode @@ -51,7 +57,7 @@ module Gitlab rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) - raise IpBlacklisted if !skip_rate_limit?(login: login) && rate_limiter.banned? + raise IpBlocked if !skip_rate_limit?(login: login) && rate_limiter.banned? # `user_with_password_for_git` should be the last check # because it's the most expensive, especially when LDAP @@ -364,14 +370,8 @@ module Gitlab ] end - def available_scopes_for(current_user) - scopes = non_admin_available_scopes - - if current_user.admin? # rubocop: disable Cop/UserAdmin - scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] - end - - scopes + def available_scopes_for(resource) + available_scopes_for_resource(resource) - unavailable_scopes_for_resource(resource) end def all_available_scopes @@ -390,13 +390,40 @@ module Gitlab end def resource_bot_scopes - Gitlab::Auth::API_SCOPES + Gitlab::Auth::REPOSITORY_SCOPES + Gitlab::Auth.registry_scopes - [:read_user] + non_admin_available_scopes - [READ_USER_SCOPE] end private + def available_scopes_for_resource(resource) + case resource + when User + scopes = non_admin_available_scopes + + if resource.admin? # rubocop: disable Cop/UserAdmin + scopes += Feature.enabled?(:admin_mode_for_api) ? ADMIN_SCOPES : [SUDO_SCOPE] + end + + scopes + when Project, Group + resource_bot_scopes + else + [] + end + end + + def unavailable_scopes_for_resource(resource) + unavailable_observability_scopes_for_resource(resource) + end + + def unavailable_observability_scopes_for_resource(resource) + return [] if resource.is_a?(Group) && Gitlab::Observability.enabled?(resource) + + OBSERVABILITY_SCOPES + end + def non_admin_available_scopes - API_SCOPES + REPOSITORY_SCOPES + registry_scopes + API_SCOPES + REPOSITORY_SCOPES + registry_scopes + OBSERVABILITY_SCOPES end def find_build_by_token(token) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index c69462b12de..4a610b26290 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -225,6 +225,12 @@ module Gitlab def access_token strong_memoize(:access_token) do + # Kubernetes API OAuth header is not OauthAccessToken or PersonalAccessToken + # and should be ignored by this method. When the kubernetes API uses a different + # header, we can remove this guard + # https://gitlab.com/gitlab-org/gitlab/-/issues/406582 + next if current_request.path.starts_with? "/api/v4/internal/kubernetes/" + if try(:namespace_inheritable, :authentication) access_token_from_namespace_inheritable else diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 6c99b505797..30896637eff 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -184,6 +184,10 @@ module Gitlab options['lowercase_usernames'] end + def sync_name + options['sync_name'] + end + def name_proc if allow_username_or_email_login proc { |name| name.gsub(/@.*\z/, '') } diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 82a5aad360c..d1eede65f0c 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -76,7 +76,13 @@ module Gitlab end def get_from_auth_hash_or_info(key) - coerce_utf8(auth_hash[key]) || get_info(key) + if auth_hash.key?(key) + coerce_utf8(auth_hash[key]) + elsif auth_hash.key?(:extra) && auth_hash.extra.key?(:raw_info) && !auth_hash.extra.raw_info[key].nil? + coerce_utf8(auth_hash.extra.raw_info[key]) + else + get_info(key) + end end # Allow for configuring a custom username claim per provider from diff --git a/lib/gitlab/auth/o_auth/session.rb b/lib/gitlab/auth/o_auth/session.rb deleted file mode 100644 index 4925b107042..00000000000 --- a/lib/gitlab/auth/o_auth/session.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -# :nocov: -module Gitlab - module Auth - module OAuth - module Session - def self.create(provider, ticket) - Rails.cache.write("gitlab:#{provider}:#{ticket}", ticket, expires_in: Gitlab.config.omniauth.cas3.session_duration) - end - - def self.destroy(provider, ticket) - Rails.cache.delete("gitlab:#{provider}:#{ticket}") - end - - def self.valid?(provider, ticket) - Rails.cache.read("gitlab:#{provider}:#{ticket}").present? - end - end - end - end -end -# :nocov: diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 01e126ec2f5..3981594478d 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -233,7 +233,7 @@ module Gitlab email ||= auth_hash.email valid_username = ::Namespace.clean_path(username) - valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } + valid_username = Gitlab::Utils::Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } { name: name.strip.presence || valid_username, @@ -258,7 +258,7 @@ module Gitlab metadata = gl_user.build_user_synced_attributes_metadata if sync_profile_from_provider? - UserSyncedAttributesMetadata.syncable_attributes.each do |key| + UserSyncedAttributesMetadata.syncable_attributes(auth_hash.provider).each do |key| if auth_hash.has_attribute?(key) && gl_user.sync_attribute?(key) gl_user.public_send("#{key}=".to_sym, auth_hash.public_send(key)) # rubocop:disable GitlabSecurity/PublicSend metadata.set_attribute_synced(key, true) diff --git a/lib/gitlab/auth/otp/duo_auth.rb b/lib/gitlab/auth/otp/duo_auth.rb new file mode 100644 index 00000000000..eeae04bc08b --- /dev/null +++ b/lib/gitlab/auth/otp/duo_auth.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module DuoAuth + def duo_auth_enabled?(_user) + ::Gitlab.config.duo_auth.enabled + end + end + end + end +end diff --git a/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb new file mode 100644 index 00000000000..57bc88de175 --- /dev/null +++ b/lib/gitlab/auth/otp/strategies/duo_auth/manual_otp.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Auth + module Otp + module Strategies + module DuoAuth + class ManualOtp < Base + include Gitlab::Utils::StrongMemoize + + def validate(otp_code) + params = { username: user.username, factor: "passcode", passcode: otp_code.to_i } + response = duo_client.request('POST', "/auth/v2/auth", params) + approve_or_deny(parse_response(response)) + rescue StandardError => e + Gitlab::AppLogger.error(e) + error(e.message) + end + + private + + def duo_client + DuoApi.new(::Gitlab.config.duo_auth.integration_key, + ::Gitlab.config.duo_auth.secret_key, + ::Gitlab.config.duo_auth.hostname) + end + strong_memoize_attr :duo_client + + def parse_response(response) + Gitlab::Json.parse(response.body) + end + + def approve_or_deny(parsed_response) + result_key = parsed_response.dig('response', 'result') + if result_key.to_s == "allow" + success + else + error(message: parsed_response.dig('response', 'status_msg').to_s) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/auth/u2f_webauthn_converter.rb b/lib/gitlab/auth/u2f_webauthn_converter.rb deleted file mode 100644 index 20b5d2ddc88..00000000000 --- a/lib/gitlab/auth/u2f_webauthn_converter.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -require 'webauthn/u2f_migrator' - -module Gitlab - module Auth - class U2fWebauthnConverter - def initialize(u2f_registration) - @u2f_registration = u2f_registration - end - - def convert - now = Time.current - - converted_credential = WebAuthn::U2fMigrator.new( - app_id: Gitlab.config.gitlab.url, - certificate: u2f_registration.certificate, - key_handle: u2f_registration.key_handle, - public_key: u2f_registration.public_key, - counter: u2f_registration.counter - ).credential - - { - credential_xid: Base64.strict_encode64(converted_credential.id), - public_key: Base64.strict_encode64(converted_credential.public_key), - counter: u2f_registration.counter || 0, - name: u2f_registration.name || '', - user_id: u2f_registration.user_id, - u2f_registration_id: u2f_registration.id, - created_at: now, - updated_at: now - } - end - - private - - attr_reader :u2f_registration - end - end -end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index 322dfa74d09..3025960a8ab 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -29,7 +29,8 @@ module Gitlab "Your password expired. "\ "Please access GitLab from a web browser to update your password." else - "Your account has been blocked." + "Your request has been rejected for an unknown reason."\ + "Please contact your GitLab administrator and/or GitLab Support." end end diff --git a/lib/gitlab/avatar_cache.rb b/lib/gitlab/avatar_cache.rb index 30c8e089061..ed00a279299 100644 --- a/lib/gitlab/avatar_cache.rb +++ b/lib/gitlab/avatar_cache.rb @@ -65,7 +65,13 @@ module Gitlab keys = emails.map { |email| email_key(email) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys) + if ::Feature.enabled?(:use_pipeline_over_multikey) + redis.pipelined do |pipeline| + keys.each { |key| pipeline.unlink(key) } + end.sum + else + redis.unlink(*keys) + end end end end diff --git a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb index 82e607ac7a7..2127ce5975d 100644 --- a/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb +++ b/lib/gitlab/background_migration/backfill_admin_mode_scope_for_personal_access_tokens.rb @@ -12,14 +12,18 @@ module Gitlab end operation_name :update_all - feature_category :authentication_and_authorization + feature_category :system_access ADMIN_MODE_SCOPE = ['admin_mode'].freeze def perform each_sub_batch do |sub_batch| sub_batch.each do |token| - token.update!(scopes: (YAML.safe_load(token.scopes) + ADMIN_MODE_SCOPE).uniq.to_yaml) + existing_scopes = YAML.safe_load(token.scopes, permitted_classes: [Symbol]) + # making sure scopes are not mixed symbols and strings + stringified_scopes = existing_scopes.map(&:to_s) + + token.update!(scopes: (stringified_scopes + ADMIN_MODE_SCOPE).uniq.to_yaml) end end end diff --git a/lib/gitlab/background_migration/backfill_compliance_violations.rb b/lib/gitlab/background_migration/backfill_compliance_violations.rb new file mode 100644 index 00000000000..131b4a05e41 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_compliance_violations.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class BackfillComplianceViolations < Gitlab::BackgroundMigration::BatchedMigrationJob + feature_category :compliance_management + + def perform + # no-op. The logic is defined in EE module. + end + end + # rubocop: enable Style/Documentation + end +end + +::Gitlab::BackgroundMigration::BackfillComplianceViolations.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_design_management_repositories.rb b/lib/gitlab/background_migration/backfill_design_management_repositories.rb new file mode 100644 index 00000000000..fe57767a693 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_design_management_repositories.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill design_management_repositories table for a range of projects + class BackfillDesignManagementRepositories < BatchedMigrationJob + operation_name :backfill_design_management_repositories + feature_category :geo_replication + + def perform + each_sub_batch do |sub_batch| + backfill_design_management_repositories(sub_batch) + end + end + + def backfill_design_management_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO design_management_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb index de52629522b..878f89a8b3d 100644 --- a/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb +++ b/lib/gitlab/background_migration/backfill_integrations_enable_ssl_verification.rb @@ -40,13 +40,13 @@ module Gitlab scope :affected, -> { where(type_new: INTEGRATIONS.keys).where.not(encrypted_properties: nil) } attr_encrypted :properties, - mode: :per_attribute_iv, - key: Settings.attr_encrypted_db_key_base_32, - algorithm: 'aes-256-gcm', - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false + mode: :per_attribute_iv, + key: Settings.attr_encrypted_db_key_base_32, + algorithm: 'aes-256-gcm', + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false # Handle assignment of props with symbol keys. # To do this correctly, we need to call the method generated by attr_encrypted. diff --git a/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb new file mode 100644 index 00000000000..1a5ad1c14a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_ldap_settings.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fill container_registry_size for project_statistics + class BackfillNamespaceLdapSettings < Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_namespace_ldap_settings + feature_category :system_access + + def perform + # no-op in FOSS + end + end + end +end + +Gitlab::BackgroundMigration::BackfillNamespaceLdapSettings.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb deleted file mode 100644 index 3b8a452b855..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_children.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to set namespaces.traversal_ids in sub-batches, of all namespaces with - # a parent and not already set. - # rubocop:disable Style/Documentation - class BackfillNamespaceTraversalIdsChildren - class Namespace < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'namespaces' - - scope :base_query, -> { where.not(parent_id: nil) } - end - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - batch_query = Namespace.base_query.where(id: start_id..end_id) - batch_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - ranged_query = Namespace.unscoped.base_query.where(id: first..last) - - update_sql = <<~SQL - UPDATE namespaces - SET traversal_ids = calculated_ids.traversal_ids - FROM #{calculated_traversal_ids(ranged_query)} calculated_ids - WHERE namespaces.id = calculated_ids.id - AND namespaces.traversal_ids = '{}' - SQL - ApplicationRecord.connection.execute(update_sql) - - sleep PAUSE_SECONDS - end - - # We have to add all arguments when marking a job as succeeded as they - # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - # Calculate the ancestor path for a given set of namespaces. - def calculated_traversal_ids(batch) - <<~SQL - ( - WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( - ( - SELECT batch.id, batch.id, batch.parent_id, 1 - FROM (#{batch.to_sql}) AS batch - ) - UNION ALL - ( - SELECT cte.source_id, n.id, n.parent_id, cte.height+1 - FROM namespaces n, cte - WHERE n.id = cte.parent_id - ) - ) - SELECT flat_hierarchy.source_id as id, - array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids - FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy - GROUP BY flat_hierarchy.source_id - ) - SQL - end - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'BackfillNamespaceTraversalIdsChildren', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb b/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb deleted file mode 100644 index c69289fb91f..00000000000 --- a/lib/gitlab/background_migration/backfill_namespace_traversal_ids_roots.rb +++ /dev/null @@ -1,51 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A job to set namespaces.traversal_ids in sub-batches, of all namespaces - # without a parent and not already set. - # rubocop:disable Style/Documentation - class BackfillNamespaceTraversalIdsRoots - class Namespace < ActiveRecord::Base - include ::EachBatch - - self.table_name = 'namespaces' - - scope :base_query, -> { where(parent_id: nil) } - end - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - ranged_query = Namespace.base_query - .where(id: start_id..end_id) - .where("traversal_ids = '{}'") - - ranged_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - # The query need to be reconstructed because .each_batch modifies the default scope - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 - Namespace.unscoped - .base_query - .where(id: first..last) - .where("traversal_ids = '{}'") - .update_all('traversal_ids = ARRAY[id]') - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'BackfillNamespaceTraversalIdsRoots', - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_partitioned_table.rb b/lib/gitlab/background_migration/backfill_partitioned_table.rb new file mode 100644 index 00000000000..6479d40a930 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_partitioned_table.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Background migration to generically copy data from the given table into its corresponding partitioned table + class BackfillPartitionedTable < BatchedMigrationJob + operation_name :upsert_partitioned_table + feature_category :database + job_arguments :partitioned_table + + def perform + validate_paritition_table! + + bulk_copy = Gitlab::Database::PartitioningMigrationHelpers::BulkCopy.new( + batch_table, + partitioned_table, + batch_column, + connection: connection + ) + + each_sub_batch do |relation| + sub_start_id, sub_stop_id = relation.pick(Arel.sql("MIN(#{batch_column}), MAX(#{batch_column})")) + bulk_copy.copy_between(sub_start_id, sub_stop_id) + end + end + + private + + def validate_paritition_table! + unless connection.table_exists?(partitioned_table) + raise "exiting backfill migration because partitioned table #{partitioned_table} does not exist. " \ + "This could be due to rollback of the migration which created the partitioned table." + end + + # rubocop: disable Style/GuardClause + unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(partitioned_table).present? + raise "exiting backfill migration because the given destination table is not partitioned." + end + # rubocop: enable Style/GuardClause + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb new file mode 100644 index 00000000000..9bf503bd6e7 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_prepared_at_merge_requests.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill prepared_at for an array of merge requests + class BackfillPreparedAtMergeRequests < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation } + operation_name :update_all + feature_category :code_review_workflow + + def perform + each_sub_batch do |sub_batch| + sub_batch.where(prepared_at: nil).where.not(merge_status: 'preparing').update_all('prepared_at = created_at') + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb new file mode 100644 index 00000000000..8d6df905f15 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_wiki_repositories.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfill project_wiki_repositories table for a range of projects + class BackfillProjectWikiRepositories < BatchedMigrationJob + operation_name :backfill_project_wiki_repositories + feature_category :geo_replication + + scope_to ->(relation) do + relation + .joins('LEFT OUTER JOIN project_wiki_repositories ON project_wiki_repositories.project_id = projects.id') + .where(project_wiki_repositories: { project_id: nil }) + end + + def perform + each_sub_batch do |sub_batch| + backfill_project_wiki_repositories(sub_batch) + end + end + + def backfill_project_wiki_repositories(relation) + connection.execute( + <<~SQL + INSERT INTO project_wiki_repositories (project_id, created_at, updated_at) + SELECT projects.id, now(), now() + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (project_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb b/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb deleted file mode 100644 index 3bf6bf993dd..00000000000 --- a/lib/gitlab/background_migration/backfill_upvotes_count_on_issues.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class that will populate the upvotes_count field - # for each issue - class BackfillUpvotesCountOnIssues - BATCH_SIZE = 1_000 - - def perform(start_id, stop_id) - (start_id..stop_id).step(BATCH_SIZE).each do |offset| - update_issue_upvotes_count(offset, offset + BATCH_SIZE) - end - end - - private - - def execute(sql) - @connection ||= ApplicationRecord.connection - @connection.execute(sql) - end - - def update_issue_upvotes_count(batch_start, batch_stop) - execute(<<~SQL) - UPDATE issues - SET upvotes_count = sub_q.count_all - FROM ( - SELECT COUNT(*) AS count_all, e.awardable_id AS issue_id - FROM award_emoji AS e - WHERE e.name = 'thumbsup' AND - e.awardable_type = 'Issue' AND - e.awardable_id BETWEEN #{batch_start} AND #{batch_stop} - GROUP BY issue_id - ) AS sub_q - WHERE sub_q.issue_id = issues.id; - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_user_namespace.rb b/lib/gitlab/background_migration/backfill_user_namespace.rb deleted file mode 100644 index df6b1f083c3..00000000000 --- a/lib/gitlab/background_migration/backfill_user_namespace.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills the `namespaces.type` column, replacing any - # instances of `NULL` with `User` - class BackfillUserNamespace - include Gitlab::Database::DynamicModelHelpers - - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id) - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size, order_hint: :type) do |sub_batch| - batch_metrics.time_operation(:update_all) do - sub_batch.update_all(type: 'User') - end - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) - end - end - - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - - private - - def connection - ApplicationRecord.connection - end - - def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id) - define_batchable_model(source_table, connection: connection) - .where(source_key_column => start_id..stop_id) - .where(type: nil) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index fc0d0ce3a57..8e2e588e0cd 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -11,7 +11,7 @@ module Gitlab class MigrationIssue < ApplicationRecord self.table_name = 'issues' - scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } + scope :base_query, ->(base_type) { where(issue_type: base_type) } end MAX_UPDATE_RETRIES = 3 @@ -24,9 +24,7 @@ module Gitlab operation_name :update_all def perform - each_sub_batch( - batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } - ) do |sub_batch| + each_sub_batch do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) # The query need to be reconstructed because .each_batch modifies the default scope diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 4039a79cfa7..952e6d01f1a 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -7,6 +7,8 @@ module Gitlab # # Job arguments needed must be defined explicitly, # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#job-arguments. + # rubocop:disable Metrics/ClassLength + # rubocop:disable Metrics/ParameterLists class BatchedMigrationJob include Gitlab::Database::DynamicModelHelpers include Gitlab::ClassAttributes @@ -60,7 +62,8 @@ module Gitlab end def initialize( - start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection: + start_id:, end_id:, batch_table:, batch_column:, sub_batch_size:, pause_ms:, job_arguments: [], connection:, + sub_batch_exception: nil ) @start_id = start_id @@ -71,6 +74,7 @@ module Gitlab @pause_ms = pause_ms @job_arguments = job_arguments @connection = connection + @sub_batch_exception = sub_batch_exception end def filter_batch(relation) @@ -87,7 +91,8 @@ module Gitlab private - attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection + attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, + :pause_ms, :connection, :sub_batch_exception def each_sub_batch(batching_arguments: {}, batching_scope: nil) all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) @@ -98,6 +103,10 @@ module Gitlab sub_batch_relation.each_batch(**all_batching_arguments) do |relation| batch_metrics.instrument_operation(operation_name) do yield relation + rescue *Gitlab::Database::BackgroundMigration::BatchedJob::TIMEOUT_EXCEPTIONS => exception + exception_class = sub_batch_exception || exception.class + + raise exception_class, exception end sleep([pause_ms, 0].max * 0.001) @@ -137,3 +146,5 @@ module Gitlab end end end +# rubocop:enable Metrics/ClassLength +# rubocop:enable Metrics/ParameterLists diff --git a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb b/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb deleted file mode 100644 index 4da120769a0..00000000000 --- a/lib/gitlab/background_migration/cleanup_orphaned_lfs_objects_projects.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The migration is used to cleanup orphaned lfs_objects_projects in order to - # introduce valid foreign keys to this table - class CleanupOrphanedLfsObjectsProjects - # A model to access lfs_objects_projects table in migrations - class LfsObjectsProject < ActiveRecord::Base - self.table_name = 'lfs_objects_projects' - - include ::EachBatch - - belongs_to :lfs_object - belongs_to :project - end - - # A model to access lfs_objects table in migrations - class LfsObject < ActiveRecord::Base - self.table_name = 'lfs_objects' - end - - # A model to access projects table in migrations - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - SUB_BATCH_SIZE = 5000 - CLEAR_CACHE_DELAY = 1.minute - - def perform(start_id, end_id) - cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) - cleanup_lfs_objects_projects_without_project(start_id, end_id) - end - - private - - def cleanup_lfs_objects_projects_without_lfs_object(start_id, end_id) - each_record_without_association(start_id, end_id, :lfs_object, :lfs_objects) do |lfs_objects_projects_without_lfs_objects| - projects = Project.where(id: lfs_objects_projects_without_lfs_objects.select(:project_id)) - - if projects.present? - ProjectCacheWorker.bulk_perform_in_with_contexts( - CLEAR_CACHE_DELAY, - projects, - arguments_proc: ->(project) { [project.id, [], [:lfs_objects_size]] }, - context_proc: ->(project) { { project: project } } - ) - end - - lfs_objects_projects_without_lfs_objects.delete_all - end - end - - def cleanup_lfs_objects_projects_without_project(start_id, end_id) - each_record_without_association(start_id, end_id, :project, :projects) do |lfs_objects_projects_without_projects| - lfs_objects_projects_without_projects.delete_all - end - end - - def each_record_without_association(start_id, end_id, association, table_name) - batch = LfsObjectsProject.where(id: start_id..end_id) - - batch.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(lfs_objects_projects.id), max(lfs_objects_projects.id)')) - - lfs_objects_without_association = - LfsObjectsProject - .unscoped - .left_outer_joins(association) - .where(id: (first..last), table_name => { id: nil }) - - yield lfs_objects_without_association - end - end - end - end -end diff --git a/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb new file mode 100644 index 00000000000..e8ee2a4c251 --- /dev/null +++ b/lib/gitlab/background_migration/cleanup_personal_access_tokens_with_nil_expires_at.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Clean up personal access tokens with expires_at value is nil + # and set the value to new default 365 days + class CleanupPersonalAccessTokensWithNilExpiresAt < BatchedMigrationJob + feature_category :system_access + + EXPIRES_AT_DEFAULT = 365.days.from_now + + scope_to ->(relation) { relation.where(expires_at: nil) } + operation_name :update_all + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all(expires_at: EXPIRES_AT_DEFAULT) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/create_vulnerability_links.rb b/lib/gitlab/background_migration/create_vulnerability_links.rb new file mode 100644 index 00000000000..bbc71dfb392 --- /dev/null +++ b/lib/gitlab/background_migration/create_vulnerability_links.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# rubocop:disable Style/Documentation +module Gitlab + module BackgroundMigration + class CreateVulnerabilityLinks < BatchedMigrationJob + feature_category :vulnerability_management + def perform; end + end + end +end + +Gitlab::BackgroundMigration::CreateVulnerabilityLinks.prepend_mod +# rubocop:enable Style/Documentation diff --git a/lib/gitlab/background_migration/delete_orphaned_deployments.rb b/lib/gitlab/background_migration/delete_orphaned_deployments.rb deleted file mode 100644 index 4a3a12ab53d..00000000000 --- a/lib/gitlab/background_migration/delete_orphaned_deployments.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for deleting orphaned deployments. - class DeleteOrphanedDeployments - include Database::MigrationHelpers - - def perform(start_id, end_id) - orphaned_deployments - .where(id: start_id..end_id) - .delete_all - - mark_job_as_succeeded(start_id, end_id) - end - - def orphaned_deployments - define_batchable_model('deployments', connection: ApplicationRecord.connection) - .where('NOT EXISTS (SELECT 1 FROM environments WHERE deployments.environment_id = environments.id)') - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb new file mode 100644 index 00000000000..a795300fa9d --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_packages_dependencies.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Deletes orphaned packages_dependencies records that have no packages_dependency_links + class DeleteOrphanedPackagesDependencies < BatchedMigrationJob + operation_name :delete_all + feature_category :package_registry + + scope_to ->(relation) { + relation.where( + <<~SQL.squish + NOT EXISTS ( + SELECT 1 + FROM packages_dependency_links + WHERE packages_dependency_links.dependency_id = packages_dependencies.id + ) + SQL + ) + } + + def perform + each_sub_batch(&:delete_all) + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb b/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb deleted file mode 100644 index dad5da875ab..00000000000 --- a/lib/gitlab/background_migration/disable_expiration_policies_linked_to_no_container_images.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - BATCH_SIZE = 1000 - - # This background migration disables container expiration policies connected - # to a project that has no container repositories - class DisableExpirationPoliciesLinkedToNoContainerImages - # rubocop: disable Style/Documentation - class ContainerExpirationPolicy < ActiveRecord::Base - include EachBatch - - self.table_name = 'container_expiration_policies' - end - # rubocop: enable Style/Documentation - - def perform(from_id, to_id) - ContainerExpirationPolicy.where(enabled: true, project_id: from_id..to_id).each_batch(of: BATCH_SIZE) do |batch| - sql = <<-SQL - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:project_id).limit(BATCH_SIZE).to_sql}) - UPDATE container_expiration_policies - SET enabled = FALSE - FROM batched_relation - WHERE container_expiration_policies.project_id = batched_relation.project_id - AND NOT EXISTS (SELECT 1 FROM "container_repositories" WHERE container_repositories.project_id = container_expiration_policies.project_id) - SQL - execute(sql) - end - end - - private - - def execute(sql) - ApplicationRecord - .connection - .execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index 2eb7c5230ba..276c7a1c6fa 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -23,8 +23,9 @@ module Gitlab .joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id') .joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id') .joins('LEFT OUTER JOIN issues ON issues.project_id = projects.id') - .where('project_statistics.repository_size' => 0, - 'project_settings.legacy_open_source_license_available' => true) + .where( + 'project_statistics.repository_size' => 0, + 'project_settings.legacy_open_source_license_available' => true) .group('projects.id') .having('COUNT(issues.id) = 0') diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 8953836c705..7661ae4b5ad 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -23,8 +23,9 @@ module Gitlab .joins('LEFT OUTER JOIN project_statistics ON project_statistics.project_id = projects.id') .joins('LEFT OUTER JOIN project_settings ON project_settings.project_id = projects.id') .joins('LEFT OUTER JOIN project_authorizations ON project_authorizations.project_id = projects.id') - .where('project_statistics.repository_size' => 0, - 'project_settings.legacy_open_source_license_available' => true) + .where( + 'project_statistics.repository_size' => 0, + 'project_settings.legacy_open_source_license_available' => true) .group('projects.id') .having('COUNT(project_authorizations.user_id) = 1') diff --git a/lib/gitlab/background_migration/drop_invalid_remediations.rb b/lib/gitlab/background_migration/drop_invalid_remediations.rb deleted file mode 100644 index f0a0de586f5..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_remediations.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class DropInvalidRemediations - def perform(start_id, stop_id) - end - end - # rubocop: enable Style/Documentation - end -end - -Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations') diff --git a/lib/gitlab/background_migration/drop_invalid_security_findings.rb b/lib/gitlab/background_migration/drop_invalid_security_findings.rb deleted file mode 100644 index 000628e109c..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_security_findings.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module BackgroundMigration - # Drop rows from security_findings where the uuid is NULL - class DropInvalidSecurityFindings - # rubocop:disable Style/Documentation - class SecurityFinding < ActiveRecord::Base - include ::EachBatch - self.table_name = 'security_findings' - scope :no_uuid, -> { where(uuid: nil) } - end - # rubocop:enable Style/Documentation - - PAUSE_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - ranged_query = SecurityFinding - .where(id: start_id..end_id) - .no_uuid - - ranged_query.each_batch(of: sub_batch_size) do |sub_batch| - first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) - - # The query need to be reconstructed because .each_batch modifies the default scope - # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 - SecurityFinding.unscoped - .where(id: first..last) - .no_uuid - .delete_all - - sleep PAUSE_SECONDS - end - - mark_job_as_succeeded(start_id, end_id, sub_batch_size) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb b/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb deleted file mode 100644 index 293530f6536..00000000000 --- a/lib/gitlab/background_migration/drop_invalid_vulnerabilities.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::DropInvalidVulnerabilities - # rubocop: disable Gitlab/NamespacedClass - class Vulnerability < ActiveRecord::Base - self.table_name = "vulnerabilities" - has_many :findings, class_name: 'VulnerabilitiesFinding', inverse_of: :vulnerability - end - - class VulnerabilitiesFinding < ActiveRecord::Base - self.table_name = "vulnerability_occurrences" - belongs_to :vulnerability, class_name: 'Vulnerability', inverse_of: :findings, foreign_key: 'vulnerability_id' - end - # rubocop: enable Gitlab/NamespacedClass - - # rubocop: disable CodeReuse/ActiveRecord - def perform(start_id, end_id) - Vulnerability - .where(id: start_id..end_id) - .left_joins(:findings) - .where(vulnerability_occurrences: { vulnerability_id: nil }) - .delete_all - - mark_job_as_succeeded(start_id, end_id) - end - # rubocop: enable CodeReuse/ActiveRecord - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'DropInvalidVulnerabilities', - arguments - ) - end -end diff --git a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb index b6e22e481fa..237c655a48a 100644 --- a/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb +++ b/lib/gitlab/background_migration/encrypt_ci_trigger_token.rb @@ -18,8 +18,7 @@ module Gitlab mode: :per_attribute_iv, algorithm: 'aes-256-gcm', key: Settings.attr_encrypted_db_key_base_32, - encode: false, - encode_vi: false + encode: false before_save :copy_token_to_encrypted_token diff --git a/lib/gitlab/background_migration/encrypt_integration_properties.rb b/lib/gitlab/background_migration/encrypt_integration_properties.rb index c9582da2a51..28c28ae48eb 100644 --- a/lib/gitlab/background_migration/encrypt_integration_properties.rb +++ b/lib/gitlab/background_migration/encrypt_integration_properties.rb @@ -18,14 +18,14 @@ module Gitlab scope :for_batch, ->(range) { where(id: range) } attr_encrypted :encrypted_properties_tmp, - attribute: :encrypted_properties, - mode: :per_attribute_iv, - key: ::Settings.attr_encrypted_db_key_base_32, - algorithm: ALGORITHM, - marshal: true, - marshaler: ::Gitlab::Json, - encode: false, - encode_iv: false + attribute: :encrypted_properties, + mode: :per_attribute_iv, + key: ::Settings.attr_encrypted_db_key_base_32, + algorithm: ALGORITHM, + marshal: true, + marshaler: ::Gitlab::Json, + encode: false, + encode_iv: false # See 'Integration#reencrypt_properties' def encrypt_properties diff --git a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb b/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb deleted file mode 100644 index 31b5b5cdb73..00000000000 --- a/lib/gitlab/background_migration/extract_project_topics_into_separate_table.rb +++ /dev/null @@ -1,63 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to extract the project topics into a separate `topics` table - class ExtractProjectTopicsIntoSeparateTable - # Temporary AR table for tags - class Tag < ActiveRecord::Base - self.table_name = 'tags' - end - - # Temporary AR table for taggings - class Tagging < ActiveRecord::Base - self.table_name = 'taggings' - belongs_to :tag - end - - # Temporary AR table for topics - class Topic < ActiveRecord::Base - self.table_name = 'topics' - end - - # Temporary AR table for project topics - class ProjectTopic < ActiveRecord::Base - self.table_name = 'project_topics' - belongs_to :topic - end - - # Temporary AR table for projects - class Project < ActiveRecord::Base - self.table_name = 'projects' - end - - def perform(start_id, stop_id) - Tagging.includes(:tag).where(taggable_type: 'Project', id: start_id..stop_id).each do |tagging| - if Project.exists?(id: tagging.taggable_id) && tagging.tag - begin - topic = Topic.find_or_create_by(name: tagging.tag.name) - project_topic = ProjectTopic.find_or_create_by(project_id: tagging.taggable_id, topic: topic) - - tagging.delete if project_topic.persisted? - rescue StandardError => e - Gitlab::ErrorTracking.log_exception(e, tagging_id: tagging.id) - end - else - tagging.delete - end - end - - mark_job_as_succeeded(start_id, stop_id) - end - - private - - def mark_job_as_succeeded(*arguments) - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( - self.class.name.demodulize, - arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb index 4b6bb12c91b..afd5e18ed7d 100644 --- a/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb +++ b/lib/gitlab/background_migration/fix_incoherent_packages_size_on_project_statistics.rb @@ -69,14 +69,14 @@ module Gitlab self.table_name = 'packages_packages' has_many :package_files, - class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' # rubocop:disable Layout/LineLength + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::PackageFile' end class PackageFile < ::ApplicationRecord self.table_name = 'packages_package_files' belongs_to :package, - class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' # rubocop:disable Layout/LineLength + class_name: '::Gitlab::BackgroundMigration::FixIncoherentPackagesSizeOnProjectStatistics::Package' def self.sum_query packages = FixIncoherentPackagesSizeOnProjectStatistics::Package.arel_table diff --git a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb deleted file mode 100644 index 4df55a7b02a..00000000000 --- a/lib/gitlab/background_migration/fix_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Background migration for fixing merge_request_diff_commit rows that don't - # have committer/author details due to - # https://gitlab.com/gitlab-org/gitlab/-/issues/344080. - class FixMergeRequestDiffCommitUsers - BATCH_SIZE = 100 - - def initialize - @commits = {} - @users = {} - end - - def perform(project_id) - # No-op, see https://gitlab.com/gitlab-org/gitlab/-/issues/344540 - end - end - end -end diff --git a/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb new file mode 100644 index 00000000000..5b3b5642ba8 --- /dev/null +++ b/lib/gitlab/background_migration/fix_vulnerability_reads_has_issues.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration fixes existing `vulnerability_reads` records which did not have `has_issues` + # correctly set at the time of creation. + class FixVulnerabilityReadsHasIssues < BatchedMigrationJob + operation_name :fix_has_issues + feature_category :vulnerability_management + + # rubocop:disable Style/Documentation + class VulnerabilityRead < ::ApplicationRecord + self.table_name = 'vulnerability_reads' + + scope :with_vulnerability_ids, ->(ids) { where(vulnerability_id: ids) } + scope :without_issues, -> { where(has_issues: false) } + end + # rubocop:enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + vulnerability_reads_with_issue_links(sub_batch).update_all('has_issues = true') + end + end + + private + + def vulnerability_reads_with_issue_links(sub_batch) + VulnerabilityRead.with_vulnerability_ids(sub_batch.select(:vulnerability_id)).without_issues + end + end + end +end diff --git a/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb new file mode 100644 index 00000000000..21ca4392003 --- /dev/null +++ b/lib/gitlab/background_migration/issues_internal_id_scope_updater.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates internal_ids records for `usage: issues` from project to namespace scope. + # For project issues it will be project namespace, for group issues it will be group namespace. + class IssuesInternalIdScopeUpdater < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :issues_internal_id_scope_updater + feature_category :database + + ISSUES_USAGE = 0 # see Enums::InternalId#usage_resources[:issues] + + scope_to ->(relation) do + relation.where(usage: ISSUES_USAGE).where.not(project_id: nil) + end + + def perform + each_sub_batch do |sub_batch| + create_namespace_scoped_records(sub_batch) + delete_project_scoped_records(sub_batch) + end + end + + private + + def delete_project_scoped_records(sub_batch) + # There is no need to keep the project scoped issues usage as we move to scoping issues to namespace. + # Also in case we do decide to move back to scoping issues usage to project, we are better off if the + # project record is not present as that would result in overlapping IIDs because project scoped issues + # usage will have outdated IIDs left in the DB + log_info("Deleted internal_ids records", ids: sub_batch.pluck(:id)) + + connection.execute( + <<~SQL + DELETE FROM internal_ids WHERE id IN (#{sub_batch.select(:id).to_sql}) + SQL + ) + end + + def create_namespace_scoped_records(sub_batch) + # Creates a corresponding namespace scoped record for every `issues` usage scoped to a project. + # On conflict it means the record was already created when a new issue is created with the + # newly namespace scoped Issue model, see Issue#has_internal_id definition. In which case to + # make sure we have the namespace_id scoped record set to the greatest of the two last_values. + created_records_ids = connection.execute( + <<~SQL + INSERT INTO internal_ids (usage, last_value, namespace_id) + SELECT #{ISSUES_USAGE}, last_value, project_namespace_id + FROM internal_ids + INNER JOIN projects ON projects.id = internal_ids.project_id + WHERE internal_ids.id IN(#{sub_batch.select(:id).to_sql}) + ON CONFLICT (usage, namespace_id) WHERE namespace_id IS NOT NULL + DO UPDATE SET last_value = GREATEST(EXCLUDED.last_value, internal_ids.last_value) + RETURNING id; + SQL + ) + + log_info("Created/updated internal_ids records", ids: created_records_ids.field_values('id')) + end + + def log_info(message, **extra) + ::Gitlab::BackgroundMigration::Logger.info(migrator: self.class.to_s, message: message, **extra) + end + end + end +end diff --git a/lib/gitlab/background_migration/logger.rb b/lib/gitlab/background_migration/logger.rb index 4ea89771eff..d338c214140 100644 --- a/lib/gitlab/background_migration/logger.rb +++ b/lib/gitlab/background_migration/logger.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Logger that can be used for migrations logging class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'migrations' end diff --git a/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb new file mode 100644 index 00000000000..dd9fcf7fcfe --- /dev/null +++ b/lib/gitlab/background_migration/migrate_evidences_for_vulnerability_findings.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the evidence data into their own records from the json attribute + class MigrateEvidencesForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_evidences_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::Finding::Evidence + class Evidence < ApplicationRecord + self.table_name = 'vulnerability_finding_evidences' + + # This data has been already validated when parsed into vulnerability_occurrences.raw_metadata + # Having this validation is a requerment from: + # https://gitlab.com/gitlab-org/gitlab/-/blob/dc3262f850cbd0ac14171d3c389b1258b4749cda/spec/db/schema_spec.rb#L253-265 + validates :data, json_schema: { filename: "filename" }, if: false + end + + def perform + each_sub_batch do |sub_batch| + migrate_evidences(sub_batch) + end + end + + private + + def migrate_evidences(sub_batch) + attrs = sub_batch.filter_map do |finding| + evidence = extract_evidence(finding.raw_metadata) + + next unless evidence + + build_evidence(finding, evidence) + end.compact + + create_evidences(attrs) if attrs.present? + end + + def build_evidence(finding, evidence) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + data: evidence, + created_at: current_time, + updated_at: current_time + } + end + + def create_evidences(evidences) + Evidence.upsert_all(evidences, returning: false, unique_by: %i[vulnerability_occurrence_id]) + end + + def extract_evidence(metadata) + # This is required because postgres doesn't support the null unicode character, i.e., \u0000. + # The following is the actual error: + # PG::UntranslatableCharacter: ERROR: unsupported Unicode escape sequence + # DETAIL: \u0000 cannot be converted to text. + return if metadata.include?('\u0000') + + parsed_metadata = Gitlab::Json.parse(metadata) + + parsed_metadata['evidence'] + rescue JSON::ParserError + nil + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_human_user_type.rb b/lib/gitlab/background_migration/migrate_human_user_type.rb new file mode 100644 index 00000000000..2cb27225274 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_human_user_type.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Migrates all users with user_type = nil to user_type = 0 + class MigrateHumanUserType < BatchedMigrationJob + OLD_TYPE_VALUE = nil + NEW_TYPE_VALUE = 0 + + operation_name :migrate_human_user_type + scope_to ->(relation) { relation.where(user_type: OLD_TYPE_VALUE) } + feature_category :user_management + + def perform + cleanup_gin_indexes('users') + + each_sub_batch do |sub_batch| + sub_batch.update_all(user_type: NEW_TYPE_VALUE) + end + end + + private + + def cleanup_gin_indexes(table_name) + sql = <<-SQL + SELECT indexname::text FROM pg_indexes WHERE tablename = '#{table_name}' AND indexdef ILIKE '%using gin%' + SQL + + index_names = ApplicationRecord.connection.select_values(sql) + + index_names.each do |index_name| + ApplicationRecord.connection.execute("SELECT gin_clean_pending_list('#{index_name}')") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb new file mode 100644 index 00000000000..0b79bc143db --- /dev/null +++ b/lib/gitlab/background_migration/migrate_links_for_vulnerability_findings.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to migrate the link data into their own records from the json attribute + class MigrateLinksForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_links_for_vulnerability_findings + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingLink + class Link < ApplicationRecord + self.table_name = 'vulnerability_finding_links' + end + + def perform + each_sub_batch(batching_scope: ->(relation) { relation.select(:id, :raw_metadata) }) do |findings| + migrate_remediations( + findings, + Link + .where(vulnerability_occurrence_id: findings.map(&:id)) + .group(:vulnerability_occurrence_id, :name, :url) + .count + ) + end + end + + private + + def migrate_remediations(findings, existing_links) + findings.each do |finding| + create_links(build_links_from(finding, existing_links)) + rescue ActiveRecord::StatementInvalid => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def build_link(finding, link) + current_time = Time.current + { + vulnerability_occurrence_id: finding.id, + name: link['name'], + url: link['url'], + created_at: current_time, + updated_at: current_time + } + end + + def build_links_from(finding, existing_links) + extract_links(finding.raw_metadata).filter_map do |link| + key = [finding.id, link['name'], link['url']] + build_link(finding, link) unless existing_links.key?(key) + end + end + + def create_links(attributes) + return if attributes.empty? + + Link.upsert_all(attributes, returning: false) + end + + def extract_links(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + parsed_links = Array.wrap(parsed_metadata['links']) + + return [] if parsed_links.blank? + + parsed_links.select { |link| link.try(:[], 'url').present? }.uniq + rescue JSON::ParserError => e + logger.warn( + message: e.message, + class: self.class.name + ) + [] + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb deleted file mode 100644 index 7d150b9cd83..00000000000 --- a/lib/gitlab/background_migration/migrate_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,296 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Migrates author and committer names and emails from - # merge_request_diff_commits to two columns that point to - # merge_request_diff_commit_users. - # - # rubocop: disable Metrics/ClassLength - class MigrateMergeRequestDiffCommitUsers - # The number of user rows in merge_request_diff_commit_users to get in a - # single query. - USER_ROWS_PER_QUERY = 1_000 - - # The number of rows in merge_request_diff_commits to get in a single - # query. - COMMIT_ROWS_PER_QUERY = 1_000 - - # The number of rows in merge_request_diff_commits to update in a single - # query. - # - # Tests in staging revealed that increasing the number of updates per - # query translates to a longer total runtime for a migration. For example, - # given the same range of rows to migrate, 1000 updates per query required - # a total of roughly 15 seconds. On the other hand, 5000 updates per query - # required a total of roughly 25 seconds. For this reason, we use a value - # of 1000 rows per update. - UPDATES_PER_QUERY = 1_000 - - # rubocop: disable Style/Documentation - class MergeRequestDiffCommit < ActiveRecord::Base - include FromUnion - extend ::SuppressCompositePrimaryKeyWarning - - self.table_name = 'merge_request_diff_commits' - - # Yields each row to migrate in the given range. - # - # This method uses keyset pagination to ensure we don't retrieve - # potentially tens of thousands (or even hundreds of thousands) of rows - # in a single query. Such queries could time out, or increase the amount - # of memory needed to process the data. - # - # We can't use `EachBatch` and similar approaches, as - # merge_request_diff_commits doesn't have a single monotonically - # increasing primary key. - def self.each_row_to_migrate(start_id, stop_id, &block) - order = Pagination::Keyset::Order.build( - %w[merge_request_diff_id relative_order].map do |col| - Pagination::Keyset::ColumnOrderDefinition.new( - attribute_name: col, - order_expression: self.arel_table[col.to_sym].asc, - nullable: :not_nullable, - distinct: false - ) - end - ) - - scope = MergeRequestDiffCommit - .where(merge_request_diff_id: start_id...stop_id) - .order(order) - - Pagination::Keyset::Iterator - .new(scope: scope, use_union_optimization: true) - .each_batch(of: COMMIT_ROWS_PER_QUERY) { |rows| rows.each(&block) } - end - end - # rubocop: enable Style/Documentation - - # rubocop: disable Style/Documentation - class MergeRequestDiffCommitUser < ActiveRecord::Base - self.table_name = 'merge_request_diff_commit_users' - - def self.union(queries) - from("(#{queries.join("\nUNION ALL\n")}) #{table_name}") - end - end - # rubocop: enable Style/Documentation - - def perform(start_id, stop_id) - return if already_processed?(start_id, stop_id) - - # This Hash maps user names + emails to their corresponding rows in - # merge_request_diff_commit_users. - user_mapping = {} - - user_details, diff_rows_to_update = get_data_to_update(start_id, stop_id) - - get_user_rows_in_batches(user_details, user_mapping) - create_missing_users(user_details, user_mapping) - update_commit_rows(diff_rows_to_update, user_mapping) - - Database::BackgroundMigrationJob.mark_all_as_succeeded( - 'MigrateMergeRequestDiffCommitUsers', - [start_id, stop_id] - ) - end - - def already_processed?(start_id, stop_id) - Database::BackgroundMigrationJob - .for_migration_execution('MigrateMergeRequestDiffCommitUsers', [start_id, stop_id]) - .succeeded - .any? - end - - # Returns the data we'll use to determine what merge_request_diff_commits - # rows to update, and what data to use for populating their - # commit_author_id and committer_id columns. - def get_data_to_update(start_id, stop_id) - # This Set is used to retrieve users that already exist in - # merge_request_diff_commit_users. - users = Set.new - - # This Hash maps the primary key of every row in - # merge_request_diff_commits to the (trimmed) author and committer - # details to use for updating the row. - to_update = {} - - MergeRequestDiffCommit.each_row_to_migrate(start_id, stop_id) do |row| - author = [prepare(row.author_name), prepare(row.author_email)] - committer = [prepare(row.committer_name), prepare(row.committer_email)] - - to_update[[row.merge_request_diff_id, row.relative_order]] = - [author, committer] - - users << author if author[0] || author[1] - users << committer if committer[0] || committer[1] - end - - [users, to_update] - end - - # Gets any existing rows in merge_request_diff_commit_users in batches. - # - # This method may end up having to retrieve lots of rows. To reduce the - # overhead, we batch queries into a UNION query. We limit the number of - # queries per UNION so we don't end up sending a single query containing - # too many SELECT statements. - def get_user_rows_in_batches(users, user_mapping) - users.each_slice(USER_ROWS_PER_QUERY) do |pairs| - queries = pairs.map do |(name, email)| - MergeRequestDiffCommitUser.where(name: name, email: email).to_sql - end - - MergeRequestDiffCommitUser.union(queries).each do |row| - user_mapping[[row.name.to_s, row.email.to_s]] = row - end - end - end - - # Creates any users for which no row exists in - # merge_request_diff_commit_users. - # - # Not all users queried may exist yet, so we need to create any missing - # ones; making sure we handle concurrent creations of the same user - def create_missing_users(users, mapping) - create = [] - - users.each do |(name, email)| - create << { name: name, email: email } unless mapping[[name, email]] - end - - return if create.empty? - - MergeRequestDiffCommitUser - .insert_all(create, returning: %w[id name email]) - .each do |row| - mapping[[row['name'], row['email']]] = MergeRequestDiffCommitUser - .new(id: row['id'], name: row['name'], email: row['email']) - end - - # It's possible for (name, email) pairs to be inserted concurrently, - # resulting in the above insert not returning anything. Here we get any - # remaining users that were created concurrently. - get_user_rows_in_batches( - users.reject { |pair| mapping.key?(pair) }, - mapping - ) - end - - # Updates rows in merge_request_diff_commits with their new - # commit_author_id and committer_id values. - def update_commit_rows(to_update, user_mapping) - to_update.each_slice(UPDATES_PER_QUERY) do |slice| - updates = {} - - slice.each do |(diff_id, order), (author, committer)| - author_id = user_mapping[author]&.id - committer_id = user_mapping[committer]&.id - - updates[[diff_id, order]] = [author_id, committer_id] - end - - bulk_update_commit_rows(updates) - end - end - - # Bulk updates rows in the merge_request_diff_commits table with their new - # author and/or committer ID values. - # - # Updates are batched together to reduce the overhead of having to produce - # a single UPDATE for every row, as we may end up having to update - # thousands of rows at once. - # - # The query produced by this method is along the lines of the following: - # - # UPDATE merge_request_diff_commits - # SET commit_author_id = - # CASE - # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN X - # WHEN ... - # END, - # committer_id = - # CASE - # WHEN (merge_request_diff_id, relative_order) = (x, y) THEN Y - # WHEN ... - # END - # WHERE (merge_request_diff_id, relative_order) IN ( (x, y), ... ) - # - # The `mapping` argument is a Hash in the following format: - # - # { [merge_request_diff_id, relative_order] => [author_id, committer_id] } - # - # rubocop: disable Metrics/AbcSize - def bulk_update_commit_rows(mapping) - author_case = Arel::Nodes::Case.new - committer_case = Arel::Nodes::Case.new - primary_values = [] - - mapping.each do |diff_id_and_order, (author_id, committer_id)| - primary_value = Arel::Nodes::Grouping.new(diff_id_and_order) - - primary_values << primary_value - - if author_id - author_case.when(primary_key.eq(primary_value)).then(author_id) - end - - if committer_id - committer_case.when(primary_key.eq(primary_value)).then(committer_id) - end - end - - if author_case.conditions.empty? && committer_case.conditions.empty? - return - end - - fields = [] - - # Statements such as `SET x = CASE END` are not valid SQL statements, so - # we omit setting an ID field if there are no values to populate it - # with. - if author_case.conditions.any? - fields << [arel_table[:commit_author_id], author_case] - end - - if committer_case.conditions.any? - fields << [arel_table[:committer_id], committer_case] - end - - query = Arel::UpdateManager.new - .table(arel_table) - .where(primary_key.in(primary_values)) - .set(fields) - .to_sql - - MergeRequestDiffCommit.connection.execute(query) - end - # rubocop: enable Metrics/AbcSize - - def primary_key - Arel::Nodes::Grouping.new( - [arel_table[:merge_request_diff_id], arel_table[:relative_order]] - ) - end - - def arel_table - MergeRequestDiffCommit.arel_table - end - - # Prepares a value to be inserted into a column in the table - # `merge_request_diff_commit_users`. Values in this table are limited to - # 512 characters. - # - # We treat empty strings as NULL values, as there's no point in (for - # example) storing a row where both the name and Email are an empty - # string. In addition, if we treated them differently we could end up with - # two rows: one where field X is NULL, and one where field X is an empty - # string. This is redundant, so we avoid storing such data. - def prepare(value) - value.present? ? value[0..511] : nil - end - end - # rubocop: enable Metrics/ClassLength - end -end diff --git a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb b/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb deleted file mode 100644 index 68bbd3cfebb..00000000000 --- a/lib/gitlab/background_migration/migrate_project_taggings_context_from_tags_to_topics.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # The class to migrate the context of project taggings from `tags` to `topics` - class MigrateProjectTaggingsContextFromTagsToTopics - # Temporary AR table for taggings - class Tagging < ActiveRecord::Base - include EachBatch - - self.table_name = 'taggings' - end - - def perform(start_id, stop_id) - Tagging.where(taggable_type: 'Project', context: 'tags', id: start_id..stop_id).each_batch(of: 500) do |relation| - relation.update_all(context: 'topics') - end - end - end - end -end diff --git a/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb new file mode 100644 index 00000000000..9eadef96db6 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_remediations_for_vulnerability_findings.rb @@ -0,0 +1,164 @@ +# frozen_string_literal: true + +module Vulnerabilities + # The class is mimicking Vulnerabilites::Remediation + class Remediation < ApplicationRecord + include FileStoreMounter + include ShaAttribute + + self.table_name = 'vulnerability_remediations' + + sha_attribute :checksum + + mount_file_store_uploader AttachmentUploader + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end + end +end + +module Gitlab + module BackgroundMigration + # The class to migrate the remediation data into their own records from the json attribute + class MigrateRemediationsForVulnerabilityFindings < BatchedMigrationJob + feature_category :vulnerability_management + operation_name :migrate_remediations_for_vulnerability_findings + + # The class to encapsulate checksum and file for uploading + class DiffFile < StringIO + # This method is used by the `carrierwave` gem + def original_filename + @original_filename ||= self.class.original_filename(checksum) + end + + def checksum + @checksum ||= self.class.checksum(string) + end + + def self.checksum(value) + Digest::SHA256.hexdigest(value) + end + + def self.original_filename(checksum) + "#{checksum}.diff" + end + end + + # The class is mimicking Vulnerabilites::Finding + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: 'vulnerability_finding_details', draft: 7 }, if: false + end + + # The class is mimicking Vulnerabilites::FindingRemediation + class FindingRemediation < ApplicationRecord + self.table_name = 'vulnerability_findings_remediations' + end + + def perform + each_sub_batch do |sub_batch| + migrate_remediations(sub_batch) + end + end + + private + + def migrate_remediations(sub_batch) + sub_batch.each do |finding| + FindingRemediation.transaction do + remediations = append_remediations_diff_checksum(finding.raw_metadata) + + result_ids = create_remediations(finding, remediations) + + create_finding_remediations(finding.id, result_ids) + end + rescue StandardError => e + logger.error( + message: e.message, + class: self.class.name, + model_id: finding.id + ) + end + end + + def create_finding_remediations(finding_id, result_ids) + attrs = result_ids.map do |result_id| + build_finding_remediation_attrs(finding_id, result_id) + end + + return unless attrs.present? + + FindingRemediation.upsert_all( + attrs, + returning: false, + unique_by: [:vulnerability_occurrence_id, :vulnerability_remediation_id] + ) + end + + def create_remediations(finding, remediations) + attrs = remediations.map do |remediation| + build_remediation_attrs(finding, remediation) + end + + return [] unless attrs.present? + + ids_checksums = ::Vulnerabilities::Remediation.upsert_all( + attrs, + returning: %w[id checksum], + unique_by: [:project_id, :checksum] + ) + + ids_checksums.each do |id_checksum| + upload_file(id_checksum['id'], id_checksum['checksum'], remediations) + end + + ids_checksums.pluck('id') + end + + def upload_file(id, checksum, remediations) + deserialized_checksum = Gitlab::Database::ShaAttribute.new.deserialize(checksum) + diff = remediations.find { |rem| rem['checksum'] == deserialized_checksum }["diff"] + file = DiffFile.new(diff) + ::Vulnerabilities::Remediation.find_by(id: id).update!(file: file) + end + + def build_remediation_attrs(finding, remediation) + { + project_id: finding.project_id, + summary: remediation['summary'], + file: DiffFile.original_filename(remediation['checksum']), + checksum: remediation['checksum'], + created_at: Time.current, + updated_at: Time.current + } + end + + def build_finding_remediation_attrs(finding_id, remediation_id) + { + vulnerability_occurrence_id: finding_id, + vulnerability_remediation_id: remediation_id, + created_at: Time.current, + updated_at: Time.current + } + end + + def append_remediations_diff_checksum(metadata) + parsed_metadata = Gitlab::Json.parse(metadata) + + return [] unless parsed_metadata['remediations'] + + parsed_metadata['remediations'].filter_map do |remediation| + next unless remediation && remediation['diff'].present? + + remediation.merge('checksum' => DiffFile.checksum(remediation['diff'])) + end.compact.uniq + end + + def logger + @logger ||= ::Gitlab::AppLogger + end + end + end +end diff --git a/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb new file mode 100644 index 00000000000..6a9f1692b72 --- /dev/null +++ b/lib/gitlab/background_migration/migrate_shared_vulnerability_identifiers.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class MigrateSharedVulnerabilityIdentifiers < BatchedMigrationJob + # rubocop: enable Style/Documentation + + feature_category :vulnerability_management + + def perform; end + end + end +end + +# rubocop: disable Layout/LineLength +Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers.prepend_mod_with("Gitlab::BackgroundMigration::MigrateSharedVulnerabilityIdentifiers") +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb b/lib/gitlab/background_migration/migrate_u2f_webauthn.rb deleted file mode 100644 index 83aa36a11e6..00000000000 --- a/lib/gitlab/background_migration/migrate_u2f_webauthn.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class MigrateU2fWebauthn - class U2fRegistration < ActiveRecord::Base - self.table_name = 'u2f_registrations' - end - - class WebauthnRegistration < ActiveRecord::Base - self.table_name = 'webauthn_registrations' - end - - def perform(start_id, end_id) - old_registrations = U2fRegistration.where(id: start_id..end_id) - old_registrations.each_slice(100) do |slice| - values = slice.map do |u2f_registration| - converter = Gitlab::Auth::U2fWebauthnConverter.new(u2f_registration) - converter.convert - end - - WebauthnRegistration.insert_all(values, unique_by: :credential_xid, returning: false) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb b/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb deleted file mode 100644 index 06422ed282f..00000000000 --- a/lib/gitlab/background_migration/move_container_registry_enabled_to_project_feature.rb +++ /dev/null @@ -1,52 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # This migration moves projects.container_registry_enabled values to - # project_features.container_registry_access_level for the projects within - # the given range of ids. - class MoveContainerRegistryEnabledToProjectFeature - MAX_BATCH_SIZE = 300 - - ENABLED = 20 - DISABLED = 0 - - def perform(from_id, to_id) - (from_id..to_id).each_slice(MAX_BATCH_SIZE) do |batch| - process_batch(batch.first, batch.last) - end - - Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded('MoveContainerRegistryEnabledToProjectFeature', [from_id, to_id]) - end - - private - - def process_batch(from_id, to_id) - ApplicationRecord.connection.execute(update_sql(from_id, to_id)) - - logger.info(message: "#{self.class}: Copied container_registry_enabled values for projects with IDs between #{from_id}..#{to_id}") - end - - # For projects that have a project_feature: - # Set project_features.container_registry_access_level to ENABLED (20) or DISABLED (0) - # depending if container_registry_enabled is true or false. - def update_sql(from_id, to_id) - <<~SQL - UPDATE project_features - SET container_registry_access_level = (CASE p.container_registry_enabled - WHEN true THEN #{ENABLED} - WHEN false THEN #{DISABLED} - ELSE #{DISABLED} - END) - FROM projects p - WHERE project_id = p.id AND - project_id BETWEEN #{from_id} AND #{to_id} - SQL - end - - def logger - @logger ||= Gitlab::BackgroundMigration::Logger.build - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb b/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb deleted file mode 100644 index 2495cb51364..00000000000 --- a/lib/gitlab/background_migration/populate_topics_total_projects_count_cache.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - SUB_BATCH_SIZE = 1_000 - - # The class to populates the total projects counter cache of topics - class PopulateTopicsTotalProjectsCountCache - # Temporary AR model for topics - class Topic < ActiveRecord::Base - include EachBatch - - self.table_name = 'topics' - end - - def perform(start_id, stop_id) - Topic.where(id: start_id..stop_id).each_batch(of: SUB_BATCH_SIZE) do |batch| - ApplicationRecord.connection.execute(<<~SQL) - WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{batch.select(:id).limit(SUB_BATCH_SIZE).to_sql}) - UPDATE topics - SET total_projects_count = (SELECT COUNT(*) FROM project_topics WHERE topic_id = batched_relation.id) - FROM batched_relation - WHERE topics.id = batched_relation.id - SQL - end - end - end - end -end diff --git a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb b/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb deleted file mode 100644 index 175966b940d..00000000000 --- a/lib/gitlab/background_migration/populate_uuids_for_security_findings.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop:disable Style/Documentation - class PopulateUuidsForSecurityFindings - NOP_RELATION = Class.new { def each_batch(*); end } - - def self.security_findings - NOP_RELATION.new - end - - def perform(*_scan_ids); end - end - end -end - -Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings.prepend_mod_with('Gitlab::BackgroundMigration::PopulateUuidsForSecurityFindings') diff --git a/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb new file mode 100644 index 00000000000..ee0f73cc3de --- /dev/null +++ b/lib/gitlab/background_migration/populate_vulnerability_dismissal_fields.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Populates missing dismissal information for vulnerabilities. + class PopulateVulnerabilityDismissalFields < BatchedMigrationJob + feature_category :vulnerability_management + scope_to ->(relation) { relation.where('state = 2 AND (dismissed_at IS NULL OR dismissed_by_id IS NULL)') } + operation_name :populate_vulnerability_dismissal_fields + + # rubocop:disable Style/Documentation + class Vulnerability < ApplicationRecord + self.table_name = 'vulnerabilities' + + has_one :finding, class_name: 'Finding' + + def copy_dismissal_information + return unless finding&.dismissal_feedback + + update_columns( + dismissed_at: finding.dismissal_feedback.created_at, + dismissed_by_id: finding.dismissal_feedback.author_id + ) + end + end + + class Finding < ApplicationRecord + self.table_name = 'vulnerability_occurrences' + + validates :details, json_schema: { filename: "filename" } + + def dismissal_feedback + Feedback.dismissal.where(finding_uuid: uuid).first + end + end + + class Feedback < ApplicationRecord + DISMISSAL_TYPE = 0 # dismissal + + self.table_name = 'vulnerability_feedback' + + scope :dismissal, -> { where(feedback_type: DISMISSAL_TYPE) } + end + # rubocop:enable Style/Documentation + + def perform + each_sub_batch do |sub_batch| + vulnerability_ids = sub_batch.pluck(:id) + Vulnerability.includes(:finding).where(id: vulnerability_ids).each do |vulnerability| + populate_for(vulnerability) + end + + log_info(vulnerability_ids) + end + end + + private + + def populate_for(vulnerability) + log_warning(vulnerability) unless vulnerability.copy_dismissal_information + rescue StandardError => error + log_error(error, vulnerability) + end + + def log_info(vulnerability_ids) + ::Gitlab::BackgroundMigration::Logger.info( + migrator: self.class.name, + message: 'Dismissal information has been copied', + count: vulnerability_ids.length + ) + end + + def log_warning(vulnerability) + ::Gitlab::BackgroundMigration::Logger.warn( + migrator: self.class.name, + message: 'Could not update vulnerability!', + vulnerability_id: vulnerability.id + ) + end + + def log_error(error, vulnerability) + ::Gitlab::BackgroundMigration::Logger.error( + migrator: self.class.name, + message: error.message, + vulnerability_id: vulnerability.id + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb deleted file mode 100644 index 15799659b55..00000000000 --- a/lib/gitlab/background_migration/remove_duplicate_vulnerabilities_findings.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -# rubocop: disable Style/Documentation -class Gitlab::BackgroundMigration::RemoveDuplicateVulnerabilitiesFindings - DELETE_BATCH_SIZE = 50 - - # rubocop:disable Gitlab/NamespacedClass - class VulnerabilitiesFinding < ActiveRecord::Base - self.table_name = "vulnerability_occurrences" - end - # rubocop:enable Gitlab/NamespacedClass - - # rubocop:disable Gitlab/NamespacedClass - class Vulnerability < ActiveRecord::Base - self.table_name = "vulnerabilities" - end - # rubocop:enable Gitlab/NamespacedClass - - def perform(start_id, end_id) - batch = VulnerabilitiesFinding.where(id: start_id..end_id) - - cte = Gitlab::SQL::CTE.new(:batch, batch.select(:report_type, :location_fingerprint, :primary_identifier_id, :project_id)) - - query = VulnerabilitiesFinding - .select('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id', 'array_agg(id) as ids') - .distinct - .with(cte.to_arel) - .from(cte.alias_to(Arel.sql('batch'))) - .joins( - %( - INNER JOIN - vulnerability_occurrences ON - vulnerability_occurrences.report_type = batch.report_type AND - vulnerability_occurrences.location_fingerprint = batch.location_fingerprint AND - vulnerability_occurrences.primary_identifier_id = batch.primary_identifier_id AND - vulnerability_occurrences.project_id = batch.project_id - )).group('batch.report_type', 'batch.location_fingerprint', 'batch.primary_identifier_id', 'batch.project_id') - .having('COUNT(*) > 1') - - ids_to_delete = [] - - query.to_a.each do |record| - # We want to keep the latest finding since it might have recent metadata - duplicate_ids = record.ids.uniq.sort - duplicate_ids.pop - ids_to_delete.concat(duplicate_ids) - - if ids_to_delete.size == DELETE_BATCH_SIZE - delete_findings_and_vulnerabilities(ids_to_delete) - ids_to_delete.clear - end - end - - delete_findings_and_vulnerabilities(ids_to_delete) if ids_to_delete.any? - end - - private - - def delete_findings_and_vulnerabilities(ids) - vulnerability_ids = VulnerabilitiesFinding.where(id: ids).pluck(:vulnerability_id).compact - VulnerabilitiesFinding.where(id: ids).delete_all - Vulnerability.where(id: vulnerability_ids).delete_all - end -end diff --git a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb index 7fe5a427d10..f4f54e2b2eb 100644 --- a/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb +++ b/lib/gitlab/background_migration/remove_occurrence_pipelines_and_duplicate_vulnerabilities_findings.rb @@ -53,7 +53,7 @@ class Gitlab::BackgroundMigration::RemoveOccurrencePipelinesAndDuplicateVulnerab def mark_job_as_succeeded(*arguments) Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( self.class.name.demodulize, - arguments + arguments ) end end diff --git a/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb new file mode 100644 index 00000000000..879e52c96bf --- /dev/null +++ b/lib/gitlab/background_migration/remove_project_group_link_with_missing_groups.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove `project_group_links` records whose associated group + # does not exist in `namespaces` table anymore. + class RemoveProjectGroupLinkWithMissingGroups < Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation } + operation_name :delete_all + feature_category :subgroups + + def perform + each_sub_batch do |sub_batch| + records = sub_batch.joins( + "LEFT OUTER JOIN namespaces ON namespaces.id = project_group_links.group_id AND namespaces.type = 'Group'" + ).where(namespaces: { id: nil }) + + ids = records.map(&:id) + + next if ids.empty? + + Gitlab::AppLogger.info({ message: 'Removing project group link with non-existent groups', + deleted_count: ids.count, + ids: ids }) + + records.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb index 0dbe2781327..56506814dc0 100644 --- a/lib/gitlab/background_migration/reset_status_on_container_repositories.rb +++ b/lib/gitlab/background_migration/reset_status_on_container_repositories.rb @@ -36,8 +36,8 @@ module Gitlab included do has_one :route, - as: :source, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' + as: :source, + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Route' end def full_path @@ -67,7 +67,7 @@ module Gitlab self.inheritance_column = :_type_disabled belongs_to :parent, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' def self.polymorphic_name 'Namespace' @@ -80,7 +80,7 @@ module Gitlab self.table_name = 'projects' belongs_to :namespace, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Namespace' alias_method :parent, :namespace alias_attribute :parent_id, :namespace_id @@ -92,7 +92,7 @@ module Gitlab self.table_name = 'container_repositories' belongs_to :project, - class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' + class_name: '::Gitlab::BackgroundMigration::ResetStatusOnContainerRepositories::Project' def tags? result = ContainerRegistry.tags_for(path).any? diff --git a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb b/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb deleted file mode 100644 index 43a7032e682..00000000000 --- a/lib/gitlab/background_migration/steal_migrate_merge_request_diff_commit_users.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # A background migration that finished any pending - # MigrateMergeRequestDiffCommitUsers jobs, and schedules new jobs itself. - # - # This migration exists so we can bypass rescheduling issues (e.g. jobs - # getting dropped after too many retries) that may occur when - # MigrateMergeRequestDiffCommitUsers jobs take longer than expected. - class StealMigrateMergeRequestDiffCommitUsers - def perform(start_id, stop_id) - MigrateMergeRequestDiffCommitUsers.new.perform(start_id, stop_id) - schedule_next_job - end - - def schedule_next_job - next_job = Database::BackgroundMigrationJob - .for_migration_class('MigrateMergeRequestDiffCommitUsers') - .pending - .first - - return unless next_job - - BackgroundMigrationWorker.perform_in( - 5.minutes, - 'StealMigrateMergeRequestDiffCommitUsers', - next_job.arguments - ) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_timelogs_project_id.rb b/lib/gitlab/background_migration/update_timelogs_project_id.rb deleted file mode 100644 index 69bb5cf6e6d..00000000000 --- a/lib/gitlab/background_migration/update_timelogs_project_id.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Class to populate project_id for timelogs - class UpdateTimelogsProjectId - BATCH_SIZE = 1000 - - def perform(start_id, stop_id) - (start_id..stop_id).step(BATCH_SIZE).each do |offset| - update_issue_timelogs(offset, offset + BATCH_SIZE) - update_merge_request_timelogs(offset, offset + BATCH_SIZE) - end - end - - def update_issue_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET project_id = issues.project_id - FROM issues - WHERE issues.id = timelogs.issue_id - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} - AND timelogs.project_id IS NULL; - SQL - end - - def update_merge_request_timelogs(batch_start, batch_stop) - execute(<<~SQL) - UPDATE timelogs - SET project_id = merge_requests.target_project_id - FROM merge_requests - WHERE merge_requests.id = timelogs.merge_request_id - AND timelogs.id BETWEEN #{batch_start} AND #{batch_stop} - AND timelogs.project_id IS NULL; - SQL - end - - def execute(sql) - @connection ||= ApplicationRecord.connection - @connection.execute(sql) - end - end - end -end diff --git a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb b/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb deleted file mode 100644 index 10db9f5064a..00000000000 --- a/lib/gitlab/background_migration/update_users_where_two_factor_auth_required_from_group.rb +++ /dev/null @@ -1,129 +0,0 @@ -# frozen_string_literal: true -# rubocop:disable Style/Documentation - -module Gitlab - module BackgroundMigration - class UpdateUsersWhereTwoFactorAuthRequiredFromGroup # rubocop:disable Metrics/ClassLength - def perform(start_id, stop_id) - ApplicationRecord.connection.execute <<~SQL - UPDATE - users - SET - require_two_factor_authentication_from_group = TRUE - WHERE - users.id BETWEEN #{start_id} - AND #{stop_id} - AND users.require_two_factor_authentication_from_group = FALSE - AND users.id IN ( - SELECT - DISTINCT users_groups_query.user_id - FROM - ( - SELECT - users.id AS user_id, - members.source_id AS group_ids - FROM - users - LEFT JOIN members ON members.source_type = 'Namespace' - AND members.requested_at IS NULL - AND members.user_id = users.id - AND members.type = 'GroupMember' - WHERE - users.require_two_factor_authentication_from_group = FALSE - AND users.id BETWEEN #{start_id} - AND #{stop_id}) AS users_groups_query - INNER JOIN LATERAL ( - WITH RECURSIVE "base_and_ancestors" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_ancestors" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = "base_and_ancestors"."parent_id" - ) - ), - "base_and_descendants" AS ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."id" = users_groups_query.group_ids - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "namespaces", - "base_and_descendants" - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."parent_id" = "base_and_descendants"."id" - ) - ) - SELECT - "namespaces".* - FROM - ( - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_ancestors" AS "namespaces" - WHERE - "namespaces"."type" = 'Group' - ) - UNION - ( - SELECT - "namespaces"."type", - "namespaces"."id", - "namespaces"."parent_id", - "namespaces"."require_two_factor_authentication" - FROM - "base_and_descendants" AS "namespaces" - WHERE - "namespaces"."type" = 'Group' - ) - ) namespaces - WHERE - "namespaces"."type" = 'Group' - AND "namespaces"."require_two_factor_authentication" = TRUE - ) AS hierarchy_tree ON TRUE - ); - SQL - end - end - end -end diff --git a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb b/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb deleted file mode 100644 index 458e0537f1c..00000000000 --- a/lib/gitlab/background_migration/update_vulnerability_occurrences_location.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # rubocop: disable Style/Documentation - class UpdateVulnerabilityOccurrencesLocation - def perform(start_id, stop_id) - end - end - # rubocop: enable Style/Documentation - end -end - -Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation.prepend_mod_with('Gitlab::BackgroundMigration::UpdateVulnerabilityOccurrencesLocation') diff --git a/lib/gitlab/backup_logger.rb b/lib/gitlab/backup_logger.rb index fad36b860ae..ec85c55d4a4 100644 --- a/lib/gitlab/backup_logger.rb +++ b/lib/gitlab/backup_logger.rb @@ -2,6 +2,8 @@ module Gitlab class BackupLogger < Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'backup_json' end diff --git a/lib/gitlab/bare_repository_import/importer.rb b/lib/gitlab/bare_repository_import/importer.rb deleted file mode 100644 index 28f1a10f9a7..00000000000 --- a/lib/gitlab/bare_repository_import/importer.rb +++ /dev/null @@ -1,132 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BareRepositoryImport - class Importer - NoAdminError = Class.new(StandardError) - - def self.execute(import_path) - unless import_path.ends_with?('/') - import_path = "#{import_path}/" - end - - repos_to_import = Dir.glob(import_path + '**/*.git') - - unless user = User.admins.order_id_asc.first - raise NoAdminError, 'No admin user found to import repositories' - end - - repos_to_import.each do |repo_path| - bare_repo = Gitlab::BareRepositoryImport::Repository.new(import_path, repo_path) - - unless bare_repo.processable? - log " * Skipping repo #{bare_repo.repo_path}".color(:yellow) - - next - end - - log "Processing #{repo_path}".color(:yellow) - - new(user, bare_repo).create_project_if_needed - end - end - - # This is called from within a rake task only used by Admins, so allow writing - # to STDOUT - def self.log(message) - puts message # rubocop:disable Rails/Output - end - - attr_reader :user, :project_name, :bare_repo - - delegate :log, to: :class - delegate :project_name, :project_full_path, :group_path, :repo_path, :wiki_path, to: :bare_repo - - def initialize(user, bare_repo) - @user = user - @bare_repo = bare_repo - end - - def create_project_if_needed - if project = Project.find_by_full_path(project_full_path) - log " * #{project.name} (#{project_full_path}) exists" - - return project - end - - create_project - end - - private - - def create_project - group = find_or_create_groups - - project = Projects::CreateService.new(user, - name: project_name, - path: project_name, - skip_disk_validation: true, - skip_wiki: bare_repo.wiki_exists?, - import_type: 'bare_repository', - namespace_id: group&.id).execute - - if project.persisted? && mv_repositories(project) - log " * Created #{project.name} (#{project_full_path})".color(:green) - - project.set_full_path - - ProjectCacheWorker.perform_async(project.id) - else - log " * Failed trying to create #{project.name} (#{project_full_path})".color(:red) - log " Errors: #{project.errors.messages}".color(:red) if project.errors.any? - end - - project - end - - def mv_repositories(project) - mv_repo(bare_repo.repo_path, project.repository) - - if bare_repo.wiki_exists? - mv_repo(bare_repo.wiki_path, project.wiki.repository) - end - - true - rescue StandardError => e - log " * Failed to move repo: #{e.message}".color(:red) - - false - end - - def mv_repo(path, repository) - repository.create_from_bundle(bundle(path)) - FileUtils.rm_rf(path) - end - - def storage_path_for_shard(shard) - Gitlab.config.repositories.storages[shard].legacy_disk_path - end - - def find_or_create_groups - return unless group_path.present? - - log " * Using namespace: #{group_path}" - - Groups::NestedCreateService.new(user, group_path: group_path).execute - end - - def bundle(repo_path) - # TODO: we could save some time and disk space by using - # `git bundle create - --all` and streaming the bundle directly to - # Gitaly, rather than writing it on disk first - bundle_path = "#{repo_path}.bundle" - cmd = %W(#{Gitlab.config.git.bin_path} --git-dir=#{repo_path} bundle create #{bundle_path} --all) - output, status = Gitlab::Popen.popen(cmd) - - raise output unless status == 0 - - bundle_path - end - end - end -end diff --git a/lib/gitlab/bare_repository_import/repository.rb b/lib/gitlab/bare_repository_import/repository.rb deleted file mode 100644 index b903c581aac..00000000000 --- a/lib/gitlab/bare_repository_import/repository.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BareRepositoryImport - class Repository - include ::Gitlab::Utils::StrongMemoize - - attr_reader :group_path, :project_name, :repo_path - - def initialize(root_path, repo_path) - unless root_path.ends_with?('/') - root_path = "#{root_path}/" - end - - @root_path = root_path - @repo_path = repo_path - - full_path = - if hashed? && !wiki? - repository.config.get('gitlab.fullpath') - else - repo_relative_path - end - - # Split path into 'all/the/namespaces' and 'project_name' - @group_path, _, @project_name = full_path.to_s.rpartition('/') - end - - def wiki_exists? - File.exist?(wiki_path) - end - - def wiki_path - @wiki_path ||= repo_path.sub(/\.git$/, '.wiki.git') - end - - def project_full_path - @project_full_path ||= "#{group_path}/#{project_name}" - end - - def processable? - return false if wiki? - return false if hashed? && (group_path.blank? || project_name.blank?) - - true - end - - private - - def wiki? - strong_memoize(:wiki) do - repo_path.end_with?('.wiki.git') - end - end - - def hashed? - strong_memoize(:hashed) do - repo_relative_path.include?('@hashed') - end - end - - def repo_relative_path - # Remove root path and `.git` at the end - repo_path[@root_path.size...-4] - end - - def repository - @repository ||= Rugged::Repository.new(repo_path) - end - end - end -end diff --git a/lib/gitlab/bitbucket_import/importer.rb b/lib/gitlab/bitbucket_import/importer.rb index 3dafe7c8962..592e75b1430 100644 --- a/lib/gitlab/bitbucket_import/importer.rb +++ b/lib/gitlab/bitbucket_import/importer.rb @@ -72,7 +72,7 @@ module Gitlab return unless last_bitbucket_issue - Issue.track_project_iid!(project, last_bitbucket_issue.iid) + Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) end def repo diff --git a/lib/gitlab/bullet/exclusions.rb b/lib/gitlab/bullet/exclusions.rb index f897ff492d9..406d0a80a07 100644 --- a/lib/gitlab/bullet/exclusions.rb +++ b/lib/gitlab/bullet/exclusions.rb @@ -27,7 +27,8 @@ module Gitlab def exclusions @exclusions ||= if File.exist?(config_file) - YAML.load_file(config_file)['exclusions']&.values || [] + config = YAML.safe_load_file(config_file, permitted_classes: [Range]) + config['exclusions']&.values || [] else [] end diff --git a/lib/gitlab/cache/client.rb b/lib/gitlab/cache/client.rb new file mode 100644 index 00000000000..37d6cac8d43 --- /dev/null +++ b/lib/gitlab/cache/client.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Cache + # It replaces Rails.cache with metrics support + class Client + DEFAULT_BACKING_RESOURCE = :unknown + + # Build Cache client with the metadata support + # + # @param cache_identifier [String] defines the location of the cache definition + # Example: "ProtectedBranches::CacheService#fetch" + # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) + # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) + # @return [Gitlab::Cache::Client] + def self.build_with_metadata( + cache_identifier:, + feature_category:, + backing_resource: DEFAULT_BACKING_RESOURCE + ) + new(Metadata.new( + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + )) + end + + def initialize(metadata, backend: Rails.cache) + @metadata = metadata + @metrics = Metrics.new(metadata) + @backend = backend + end + + def read(name) + read_result = backend.read(name) + + if read_result.nil? + metrics.increment_cache_miss + else + metrics.increment_cache_hit + end + + read_result + end + + def fetch(name, options = nil, &block) + read_result = read(name) + + return read_result unless block || read_result + + backend.fetch(name, options) do + metrics.observe_cache_generation(&block) + end + end + + delegate :write, :exist?, :delete, to: :backend + + attr_reader :metadata, :metrics + + private + + attr_reader :backend + end + end +end diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb index d6c89b5b2c3..de35b332300 100644 --- a/lib/gitlab/cache/metadata.rb +++ b/lib/gitlab/cache/metadata.rb @@ -5,21 +5,23 @@ module Gitlab # Value object for cache metadata class Metadata VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze - DEFAULT_BACKING_RESOURCE = :unknown + # @param cache_identifier [String] defines the location of the cache definition + # Example: "ProtectedBranches::CacheService#fetch" + # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) + # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) + # @return [Gitlab::Cache::Metadata] def initialize( cache_identifier:, feature_category:, - caller_id: Gitlab::ApplicationContext.current_context_attribute(:caller_id), - backing_resource: DEFAULT_BACKING_RESOURCE + backing_resource: Client::DEFAULT_BACKING_RESOURCE ) @cache_identifier = cache_identifier @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) - @caller_id = caller_id @backing_resource = fetch_backing_resource!(backing_resource) end - attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + attr_reader :cache_identifier, :feature_category, :backing_resource private @@ -28,7 +30,7 @@ module Gitlab raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? - DEFAULT_BACKING_RESOURCE + Client::DEFAULT_BACKING_RESOURCE end end end diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb index 00d4e6e4d4e..d9c80f076b9 100644 --- a/lib/gitlab/cache/metrics.rb +++ b/lib/gitlab/cache/metrics.rb @@ -58,7 +58,6 @@ module Gitlab def labels @labels ||= { - caller_id: cache_metadata.caller_id, cache_identifier: cache_metadata.cache_identifier, feature_category: cache_metadata.feature_category, backing_resource: cache_metadata.backing_resource diff --git a/lib/gitlab/changes_list.rb b/lib/gitlab/changes_list.rb index 999d2ee4356..ca55692ca2f 100644 --- a/lib/gitlab/changes_list.rb +++ b/lib/gitlab/changes_list.rb @@ -15,12 +15,12 @@ module Gitlab end def changes - @changes ||= @raw_changes.map do |change| + @changes ||= @raw_changes.filter_map do |change| next if change.blank? oldrev, newrev, ref = change.strip.split(' ') { oldrev: oldrev, newrev: newrev, ref: ref } - end.compact + end end end end diff --git a/lib/gitlab/chat/responder.rb b/lib/gitlab/chat/responder.rb index 478be5bd350..7ec64b7cfbf 100644 --- a/lib/gitlab/chat/responder.rb +++ b/lib/gitlab/chat/responder.rb @@ -11,21 +11,13 @@ module Gitlab # # build - A `Ci::Build` that executed a chat command. def self.responder_for(build) - if Feature.enabled?(:use_response_url_for_chat_responder) - response_url = build.pipeline.chat_data&.response_url - return unless response_url + response_url = build.pipeline.chat_data&.response_url + return unless response_url - if response_url.start_with?('https://hooks.slack.com/') - Gitlab::Chat::Responder::Slack.new(build) - else - Gitlab::Chat::Responder::Mattermost.new(build) - end + if response_url.start_with?('https://hooks.slack.com/') + Gitlab::Chat::Responder::Slack.new(build) else - integration = build.pipeline.chat_data&.chat_name&.integration - - if (responder = integration.try(:chat_responder)) - responder.new(build) - end + Gitlab::Chat::Responder::Mattermost.new(build) end end end diff --git a/lib/gitlab/checks/base_single_checker.rb b/lib/gitlab/checks/base_single_checker.rb index 435f4ccf5ba..755778efa60 100644 --- a/lib/gitlab/checks/base_single_checker.rb +++ b/lib/gitlab/checks/base_single_checker.rb @@ -5,7 +5,7 @@ module Gitlab class BaseSingleChecker < BaseChecker attr_reader :change_access - delegate(*SingleChangeAccess::ATTRIBUTES, to: :change_access) + delegate(*SingleChangeAccess::ATTRIBUTES, :branch_ref?, :tag_ref?, to: :change_access) def initialize(change_access) @change_access = change_access diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 99752dc6a01..194e3f6e938 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -36,13 +36,13 @@ module Gitlab # any of the new revisions. def commits strong_memoize(:commits) do - newrevs = @changes.map do |change| + newrevs = @changes.filter_map do |change| newrev = change[:newrev] next if blank_rev?(newrev) newrev - end.compact + end next [] if newrevs.empty? @@ -89,7 +89,7 @@ module Gitlab @single_changes_accesses ||= changes.map do |change| commits = - if blank_rev?(change[:newrev]) + if !commitish_ref?(change[:ref]) || blank_rev?(change[:newrev]) [] else Gitlab::Lazy.new { commits_for(change[:oldrev], change[:newrev]) } @@ -122,6 +122,14 @@ module Gitlab def blank_rev?(rev) rev.blank? || Gitlab::Git.blank_ref?(rev) end + + # refs/notes/commits contains commits added via `git-notes`. We currently + # have no features that check notes so we can skip them. To future-proof + # we are skipping anything that isn't a branch or tag ref as those are + # the only refs that can contain commits. + def commitish_ref?(ref) + Gitlab::Git.branch_ref?(ref) || Gitlab::Git.tag_ref?(ref) + end end end end diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index d8f5cec8a4a..083c2448a0a 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -10,6 +10,10 @@ module Gitlab }.freeze def validate! + # git-notes stores notes history as commits in refs/notes/commits (by + # default but is configurable) so we restrict the diff checks to tag + # and branch refs + return unless tag_ref? || branch_ref? return if deletion? return unless should_run_validations? return if commits.empty? diff --git a/lib/gitlab/checks/matching_merge_request.rb b/lib/gitlab/checks/matching_merge_request.rb index e5ce862264f..15178597a99 100644 --- a/lib/gitlab/checks/matching_merge_request.rb +++ b/lib/gitlab/checks/matching_merge_request.rb @@ -17,7 +17,7 @@ module Gitlab # # 1. Sidekiq: MergeService runs and updates the merge request in a locked state. # 2. Gitaly: The UserMergeBranch RPC runs. - # 3. Gitaly (gitaly-ruby): This RPC calls the pre-receive hook. + # 3. Gitaly: The RPC calls the pre-receive hook. # 4. Rails: This hook makes an API request to /api/v4/internal/allowed. # 5. Rails: This API check does a SQL query for locked merge # requests with a matching SHA. diff --git a/lib/gitlab/checks/single_change_access.rb b/lib/gitlab/checks/single_change_access.rb index 2fd48dfbfe2..9f427e98e55 100644 --- a/lib/gitlab/checks/single_change_access.rb +++ b/lib/gitlab/checks/single_change_access.rb @@ -14,7 +14,9 @@ module Gitlab protocol:, logger:, commits: nil ) @oldrev, @newrev, @ref = change.values_at(:oldrev, :newrev, :ref) + @branch_ref = Gitlab::Git.branch_ref?(@ref) @branch_name = Gitlab::Git.branch_name(@ref) + @tag_ref = Gitlab::Git.tag_ref?(@ref) @tag_name = Gitlab::Git.tag_name(@ref) @user_access = user_access @project = project @@ -38,6 +40,14 @@ module Gitlab @commits ||= project.repository.new_commits(newrev) end + def branch_ref? + @branch_ref + end + + def tag_ref? + @tag_ref + end + protected def ref_level_checks diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index fdd49df1e24..1d26bceb7b1 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -9,14 +9,14 @@ module Gitlab class Parser # keys represent the trailing digit in color changing command (30-37, 40-47, 90-97. 100-107) COLOR = { - 0 => 'black', # not that this is gray in the intense color table + 0 => 'black', # Note: This is gray in the intense color table. 1 => 'red', 2 => 'green', 3 => 'yellow', 4 => 'blue', 5 => 'magenta', 6 => 'cyan', - 7 => 'white' # not that this is gray in the dark (aka default) color table + 7 => 'white' # Note: This is gray in the dark (aka default) color table. }.freeze STYLE_SWITCHES = { diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index b2b6ce649ed..3aec1cde1bc 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -1,11 +1,18 @@ # frozen_string_literal: true +require 'openssl' + # In this class we keep track of the state changes that the # Converter makes as it scans through the log stream. module Gitlab module Ci module Ansi2json class State + include Gitlab::Utils::StrongMemoize + + SIGNATURE_KEY_SALT = 'gitlab-ci-ansi2json-state' + SEPARATOR = '--' + attr_accessor :offset, :current_line, :inherited_style, :open_sections, :last_line_offset def initialize(new_state, stream_size) @@ -18,12 +25,15 @@ module Gitlab end def encode - state = { + json = { offset: @last_line_offset, style: @current_line.style.to_h, open_sections: @open_sections - } - Base64.urlsafe_encode64(state.to_json) + }.to_json + + encoded = Base64.urlsafe_encode64(json, padding: false) + + encoded + SEPARATOR + sign(encoded) end def open_section(section, timestamp, options) @@ -85,14 +95,55 @@ module Gitlab end end - def decode_state(state) - return unless state.present? + def decode_state(data) + return if data.blank? - decoded_state = Base64.urlsafe_decode64(state) + encoded_state = verify(data) + if encoded_state.blank? + ::Gitlab::AppLogger.warn(message: "#{self.class}: signature missing or invalid", invalid_state: data) + return + end + + decoded_state = Base64.urlsafe_decode64(encoded_state) return unless decoded_state.present? - Gitlab::Json.parse(decoded_state) + ::Gitlab::Json.parse(decoded_state) + end + + def sign(message) + ::OpenSSL::HMAC.hexdigest( + signature_digest, + signature_key, + message + ) + end + + def verify(signed_message) + signature_length = signature_digest.digest_length * 2 # a byte is exactly two hexadecimals + message_length = signed_message.length - SEPARATOR.length - signature_length + return if message_length <= 0 + + signature = signed_message.last(signature_length) + message = signed_message.first(message_length) + return unless valid_signature?(message, signature) + + message + end + + def valid_signature?(message, signature) + expected_signature = sign(message) + expected_signature.bytesize == signature.bytesize && + ::OpenSSL.fixed_length_secure_compare(signature, expected_signature) + end + + def signature_digest + ::OpenSSL::Digest.new('SHA256') + end + + def signature_key + ::Gitlab::Application.key_generator.generate_key(SIGNATURE_KEY_SALT, signature_digest.block_length) end + strong_memoize_attr :signature_key end end end diff --git a/lib/gitlab/ci/badge/release/latest_release.rb b/lib/gitlab/ci/badge/release/latest_release.rb index e73bb2a912a..8d84a54787b 100644 --- a/lib/gitlab/ci/badge/release/latest_release.rb +++ b/lib/gitlab/ci/badge/release/latest_release.rb @@ -10,7 +10,8 @@ module Gitlab::Ci @project = project @customization = { key_width: opts[:key_width] ? opts[:key_width].to_i : nil, - key_text: opts[:key_text] + key_text: opts[:key_text], + value_width: opts[:value_width] ? opts[:value_width].to_i : nil } # In the future, we should support `order_by=semver` for showing the diff --git a/lib/gitlab/ci/badge/release/template.rb b/lib/gitlab/ci/badge/release/template.rb index 354be6276fa..549742226a1 100644 --- a/lib/gitlab/ci/badge/release/template.rb +++ b/lib/gitlab/ci/badge/release/template.rb @@ -11,9 +11,11 @@ module Gitlab::Ci }.freeze KEY_WIDTH_DEFAULT = 90 VALUE_WIDTH_DEFAULT = 54 + VALUE_WIDTH_MAXIMUM = 200 def initialize(badge) @tag = badge.tag || "none" + @value_width = badge.customization[:value_width] super end @@ -30,7 +32,11 @@ module Gitlab::Ci end def value_width - VALUE_WIDTH_DEFAULT + if @value_width && @value_width.between?(1, VALUE_WIDTH_MAXIMUM) + @value_width + else + VALUE_WIDTH_DEFAULT + end end def value_color diff --git a/lib/gitlab/ci/build/cache.rb b/lib/gitlab/ci/build/cache.rb index 1cddc9fcc98..c1052f59272 100644 --- a/lib/gitlab/ci/build/cache.rb +++ b/lib/gitlab/ci/build/cache.rb @@ -9,8 +9,10 @@ module Gitlab def initialize(cache, pipeline) cache = Array.wrap(cache) @cache = cache.map.with_index do |cache, index| + prefix = cache_prefix(cache, index) + Gitlab::Ci::Pipeline::Seed::Build::Cache - .new(pipeline, cache, index) + .new(pipeline, cache, prefix) end end @@ -23,6 +25,20 @@ module Gitlab end end end + + private + + # The below method fixes a bug related to incorrect caches being used + # For more details please see: https://gitlab.com/gitlab-org/gitlab/-/issues/388374 + def cache_prefix(cache, index) + files = cache.dig(:key, :files) if cache.is_a?(Hash) && cache[:key].is_a?(Hash) + + return index if files.blank? + + filenames = files.map { |file| file.split('.').first }.join('_') + + "#{index}_#{filenames}" + end end end end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index dee95534b07..bc7aad1b186 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -6,12 +6,14 @@ module Gitlab class Rules include ::Gitlab::Utils::StrongMemoize - Result = Struct.new(:when, :start_in, :allow_failure, :variables, :errors) do + Result = Struct.new(:when, :start_in, :allow_failure, :variables, :needs, :errors) do def build_attributes { when: self.when, options: { start_in: start_in }.compact, - allow_failure: allow_failure + allow_failure: allow_failure, + scheduling_type: (:dag if needs), + needs_attributes: needs&.[](:job) }.compact end @@ -33,13 +35,14 @@ module Gitlab matched_rule.attributes[:when] || @default_when, matched_rule.attributes[:start_in], matched_rule.attributes[:allow_failure], - matched_rule.attributes[:variables] + matched_rule.attributes[:variables], + (matched_rule.attributes[:needs] if Feature.enabled?(:introduce_rules_with_needs, pipeline.project)) ) else Result.new('never') end rescue Rule::Clause::ParseError => e - Result.new('never', nil, nil, nil, [e.message]) + Result.new('never', nil, nil, nil, nil, [e.message]) end private diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 010ce57d2a0..27a7611ffdd 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -6,6 +6,8 @@ module Gitlab class InstancePath include Gitlab::Utils::StrongMemoize + LATEST_VERSION_KEYWORD = '~latest' + def self.match?(address) address.include?('@') && address.start_with?(Settings.gitlab_ci['component_fqdn']) end @@ -39,9 +41,9 @@ module Gitlab File.join(component_dir, @content_filename).delete_prefix('/') end - # TODO: Add support when version is a released tag and "~latest" moving target def sha return unless project + return latest_version_sha if version == LATEST_VERSION_KEYWORD project.commit(version)&.id end @@ -69,6 +71,12 @@ module Gitlab ::Project.where_full_path_in(possible_paths).take # rubocop: disable CodeReuse/ActiveRecord end + + def latest_version_sha + return unless catalog_resource = project&.catalog_resource + + catalog_resource.latest_version&.sha + end end end end diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 585e671ce42..0c293c3f0ef 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize ConfigError = Class.new(StandardError) - TIMEOUT_SECONDS = 30.seconds + TIMEOUT_SECONDS = ENV.fetch('GITLAB_CI_CONFIG_FETCH_TIMEOUT_SECONDS', 30).to_i.clamp(0, 60).seconds TIMEOUT_MESSAGE = 'Request timed out when fetching configuration files.' RESCUE_ERRORS = [ @@ -21,14 +21,15 @@ module Gitlab attr_reader :root, :context, :source_ref_path, :source, :logger - def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, logger: nil) + # rubocop: disable Metrics/ParameterLists + def initialize(config, project: nil, pipeline: nil, sha: nil, user: nil, parent_pipeline: nil, source: nil, pipeline_config: nil, logger: nil) @logger = logger || ::Gitlab::Ci::Pipeline::Logger.new(project: project) @source_ref_path = pipeline&.source_ref_path @project = project @context = self.logger.instrument(:config_build_context, once: true) do pipeline ||= ::Ci::Pipeline.new(project: project, sha: sha, user: user, source: source) - build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline) + build_context(project: project, pipeline: pipeline, sha: sha, user: user, parent_pipeline: parent_pipeline, pipeline_config: pipeline_config) end @context.set_deadline(TIMEOUT_SECONDS) @@ -49,6 +50,7 @@ module Gitlab rescue *rescue_errors => e raise Config::ConfigError, e.message end + # rubocop: enable Metrics/ParameterLists def valid? @root.valid? @@ -117,8 +119,7 @@ module Gitlab def expand_config(config) build_config(config) - rescue Gitlab::Config::Loader::Yaml::DataTooLargeError, - Gitlab::Config::Loader::MultiDocYaml::DataTooLargeError => e + rescue Gitlab::Config::Loader::Yaml::DataTooLargeError => e track_and_raise_for_dev_exception(e) raise Config::ConfigError, e.message @@ -157,13 +158,14 @@ module Gitlab end end - def build_context(project:, pipeline:, sha:, user:, parent_pipeline:) + def build_context(project:, pipeline:, sha:, user:, parent_pipeline:, pipeline_config:) Config::External::Context.new( project: project, sha: sha || find_sha(project), user: user, parent_pipeline: parent_pipeline, variables: build_variables(pipeline: pipeline), + pipeline_config: pipeline_config, logger: logger) end diff --git a/lib/gitlab/ci/config/entry/cache.rb b/lib/gitlab/ci/config/entry/cache.rb index a635f409109..b3ff74c14da 100644 --- a/lib/gitlab/ci/config/entry/cache.rb +++ b/lib/gitlab/ci/config/entry/cache.rb @@ -9,11 +9,12 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[key untracked paths when policy unprotect].freeze + ALLOWED_KEYS = %i[key untracked paths when policy unprotect fallback_keys].freeze ALLOWED_POLICY = %w[pull-push push pull].freeze DEFAULT_POLICY = 'pull-push' ALLOWED_WHEN = %w[on_success on_failure always].freeze DEFAULT_WHEN = 'on_success' + DEFAULT_FALLBACK_KEYS = [].freeze validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS @@ -27,6 +28,8 @@ module Gitlab in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } + + validates :fallback_keys, length: { maximum: 5, too_long: "has to many entries (maximum %{count})" } end end @@ -42,7 +45,10 @@ module Gitlab entry :paths, Entry::Paths, description: 'Specify which paths should be cached across builds.' - attributes :policy, :when, :unprotect + entry :fallback_keys, ::Gitlab::Config::Entry::ArrayOfStrings, + description: 'List of keys to download cache from if no cache hit occurred for key' + + attributes :policy, :when, :unprotect, :fallback_keys def value result = super @@ -52,6 +58,7 @@ module Gitlab result[:policy] = policy || DEFAULT_POLICY # Use self.when to avoid conflict with reserved word result[:when] = self.when || DEFAULT_WHEN + result[:fallback_keys] = fallback_keys || DEFAULT_FALLBACK_KEYS result end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7c49b59a7f0..d31d1b366c3 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks environment coverage retry parallel interruptible timeout - release id_tokens].freeze + release id_tokens publish].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -45,6 +45,8 @@ module Gitlab errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end + + validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? } end entry :before_script, Entry::Commands, @@ -125,10 +127,14 @@ module Gitlab inherit: false, metadata: { composable_class: ::Gitlab::Ci::Config::Entry::IdToken } + entry :publish, Entry::Publish, + description: 'Path to be published with Pages', + inherit: false + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, - :release, :allow_failure + :release, :allow_failure, :publish def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -164,12 +170,13 @@ module Gitlab artifacts: artifacts_value, release: release_value, after_script: after_script_value, - hooks: hooks_pre_get_sources_script_enabled? ? hooks_value : nil, + hooks: hooks_value, ignore: ignored?, allow_failure_criteria: allow_failure_criteria, needs: needs_defined? ? needs_value : nil, scheduling_type: needs_defined? ? :dag : :stage, - id_tokens: id_tokens_value + id_tokens: id_tokens_value, + publish: publish ).compact end @@ -177,6 +184,10 @@ module Gitlab allow_failure_defined? ? static_allow_failure : manual_action? end + def pages_job? + name == :pages + end + def self.allowed_keys ALLOWED_KEYS end @@ -194,10 +205,6 @@ module Gitlab allow_failure_value end - - def hooks_pre_get_sources_script_enabled? - YamlProcessor::FeatureFlags.enabled?(:ci_hooks_pre_get_sources_script) - end end end end diff --git a/lib/gitlab/ci/config/entry/product/parallel.rb b/lib/gitlab/ci/config/entry/product/parallel.rb index e91714e3f5c..59cd3d3cf91 100644 --- a/lib/gitlab/ci/config/entry/product/parallel.rb +++ b/lib/gitlab/ci/config/entry/product/parallel.rb @@ -19,7 +19,7 @@ module Gitlab validations do validates :config, numericality: { only_integer: true, - greater_than_or_equal_to: 2, + greater_than_or_equal_to: 1, less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT }, allow_nil: true diff --git a/lib/gitlab/ci/config/entry/publish.rb b/lib/gitlab/ci/config/entry/publish.rb new file mode 100644 index 00000000000..52a2487009e --- /dev/null +++ b/lib/gitlab/ci/config/entry/publish.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the path to be published with Pages. + # + class Publish < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + + def self.default + 'public' + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/rules/rule.rb b/lib/gitlab/ci/config/entry/rules/rule.rb index 63bf1b38ac6..1e7f6056a65 100644 --- a/lib/gitlab/ci/config/entry/rules/rule.rb +++ b/lib/gitlab/ci/config/entry/rules/rule.rb @@ -9,7 +9,7 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables].freeze + ALLOWED_KEYS = %i[if changes exists when start_in allow_failure variables needs].freeze ALLOWED_WHEN = %w[on_success on_failure always never manual delayed].freeze attributes :if, :exists, :when, :start_in, :allow_failure @@ -20,6 +20,11 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables to define for rule conditions.' + entry :needs, Entry::Needs, + description: 'Needs configuration to define for rule conditions.', + metadata: { allowed_needs: %i[job] }, + inherit: false + validations do validates :config, presence: true validates :config, type: { with: Hash } @@ -46,7 +51,8 @@ module Gitlab def value config.merge( changes: (changes_value if changes_defined?), - variables: (variables_value if variables_defined?) + variables: (variables_value if variables_defined?), + needs: (needs_value if needs_defined?) ).compact end diff --git a/lib/gitlab/ci/config/external/context.rb b/lib/gitlab/ci/config/external/context.rb index 6eef279d3de..b8e012ec851 100644 --- a/lib/gitlab/ci/config/external/context.rb +++ b/lib/gitlab/ci/config/external/context.rb @@ -9,29 +9,29 @@ module Gitlab TimeoutError = Class.new(StandardError) - MAX_INCLUDES = 100 - NEW_MAX_INCLUDES = 150 # Update to MAX_INCLUDES when FF ci_includes_count_duplicates is removed + TEMP_MAX_INCLUDES = 100 # For logging; to be removed in https://gitlab.com/gitlab-org/gitlab/-/issues/396776 include ::Gitlab::Utils::StrongMemoize - attr_reader :project, :sha, :user, :parent_pipeline, :variables + attr_reader :project, :sha, :user, :parent_pipeline, :variables, :pipeline_config attr_reader :expandset, :execution_deadline, :logger, :max_includes delegate :instrument, to: :logger def initialize( project: nil, sha: nil, user: nil, parent_pipeline: nil, variables: nil, - logger: nil + pipeline_config: nil, logger: nil ) @project = project @sha = sha @user = user @parent_pipeline = parent_pipeline @variables = variables || Ci::Variables::Collection.new - @expandset = Feature.enabled?(:ci_includes_count_duplicates, project) ? [] : Set.new + @pipeline_config = pipeline_config + @expandset = [] @execution_deadline = 0 @logger = logger || Gitlab::Ci::Pipeline::Logger.new(project: project) - @max_includes = Feature.enabled?(:ci_includes_count_duplicates, project) ? NEW_MAX_INCLUDES : MAX_INCLUDES + @max_includes = Gitlab::CurrentSettings.current_application_settings.ci_max_includes yield self if block_given? end @@ -91,6 +91,13 @@ module Gitlab expandset.map(&:metadata) end + # Some Ci::ProjectConfig sources prepend the config content with an "internal" `include`, which becomes + # the first included file. When running a pipeline, we pass pipeline_config into the context of the first + # included file, which we use in this method to determine if the file is an "internal" one. + def internal_include? + !!pipeline_config&.internal_include_prepended? + end + protected attr_writer :expandset, :execution_deadline, :logger, :max_includes diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 0b90d240a15..273d78bd583 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -22,7 +22,7 @@ module Gitlab strong_memoize(:content) do Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) rescue Gitlab::Ci::ArtifactFileReader::Error => error - errors.push(error.message) + errors.push(error.message) # TODO this memoizes the error message as a content! end end diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 84f34f2584b..6b635cdf33b 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -61,16 +61,16 @@ module Gitlab [params, context.project&.full_path, context.sha].hash end - def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_hash # calling the method loads then memoizes the result - end - - context.logger.instrument(:config_file_expand_content_includes) do - expanded_content_hash # calling the method expands then memoizes the result - end + # This method is overridden to load context into the memoized result + # or to lazily load context via BatchLoader + def preload_context + # no-op + end - validate_hash! + def preload_content + # calling the `content` method either loads content into the memoized result + # or lazily loads it via BatchLoader + content end def validate_location! @@ -82,31 +82,65 @@ module Gitlab end def validate_context! - raise NotImplementedError, 'subclass must implement validate_context' + raise NotImplementedError, 'subclass must implement `validate_context!`' end def validate_content! - if content.blank? - errors.push("Included file `#{masked_location}` is empty or does not exist!") + errors.push("Included file `#{masked_location}` is empty or does not exist!") if content.blank? + end + + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_result # calling the method loads YAML then memoizes the content result + end + + context.logger.instrument(:config_file_interpolate_result) do + interpolator.interpolate! + end + + return validate_interpolation! unless interpolator.valid? + + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result end + + validate_hash! end protected - def expanded_content_hash - return unless content_hash + def content_result + ::Gitlab::Ci::Config::Yaml + .load_result!(content, project: context.project) + end + strong_memoize_attr :content_result - strong_memoize(:expanded_content_hash) do - expand_includes(content_hash) - end + def content_inputs + # TODO: remove support for `with` syntax in 16.1, see https://gitlab.com/gitlab-org/gitlab/-/issues/408369 + # In the interim prefer `inputs` over `with` while allow either syntax. + params.to_h.slice(:inputs, :with).each_value.first end + strong_memoize_attr :content_inputs def content_hash - strong_memoize(:content_hash) do - ::Gitlab::Ci::Config::Yaml.load!(content) + interpolator.interpolate! + + interpolator.to_hash + end + strong_memoize_attr :content_hash + + def interpolator + External::Interpolator + .new(content_result, content_inputs, context) + end + strong_memoize_attr :interpolator + + def expanded_content_hash + return if content_hash.blank? + + strong_memoize(:expanded_content_hash) do + expand_includes(content_hash) end - rescue Gitlab::Config::Loader::FormatError - nil end def validate_hash! @@ -115,6 +149,12 @@ module Gitlab end end + def validate_interpolation! + return if interpolator.valid? + + errors.push("`#{masked_location}`: #{interpolator.error_message}") + end + def expand_includes(hash) External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end diff --git a/lib/gitlab/ci/config/external/file/component.rb b/lib/gitlab/ci/config/external/file/component.rb index 33e7724bf9b..9679d78a1aa 100644 --- a/lib/gitlab/ci/config/external/file/component.rb +++ b/lib/gitlab/ci/config/external/file/component.rb @@ -11,11 +11,12 @@ module Gitlab def initialize(params, context) @location = params[:component] + super end def matching? - super && ::Feature.enabled?(:ci_include_components, context.project) + super && ::Feature.enabled?(:ci_include_components, context.project&.root_namespace) end def content @@ -48,9 +49,7 @@ module Gitlab end def validate_content! - return if content.present? - - errors.push(component_result.message) + errors.push(component_result.message) unless content.present? end private diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index f8d4cb27710..16a6bc8a692 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -15,7 +15,8 @@ module Gitlab # `Repository#blobs_at` does not support files with the `/` prefix. @location = Gitlab::Utils.remove_leading_slashes(params[:file]) - @project_name = get_project_name(params[:project]) + # We are using the same downcase in the `project` method. + @project_name = get_project_name(params[:project]).to_s.downcase @ref_name = params[:ref] || 'HEAD' super @@ -39,6 +40,15 @@ module Gitlab ) end + def preload_context + # + # calling these methods lazily loads them via BatchLoader + # + project + can_access_local_content? + sha + end + def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") @@ -58,21 +68,55 @@ module Gitlab private def project - strong_memoize(:project) do - ::Project.find_by_full_path(project_name) + return legacy_project if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + + # Although we use `where_full_path_in`, this BatchLoader does not reduce the number of queries to 1. + # That's because we use it in the `can_access_local_content?` and `sha` BatchLoaders + # as the `for` parameter. And this loads the project immediately. + BatchLoader.for(project_name) + .batch do |project_names, loader| + ::Project.where_full_path_in(project_names.uniq).each do |project| + # We are using the same downcase in the `initialize` method. + loader.call(project.full_path.downcase, project) + end end end def can_access_local_content? - strong_memoize(:can_access_local_content) do - context.logger.instrument(:config_file_project_validate_access) do - Ability.allowed?(context.user, :download_code, project) + if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + return legacy_can_access_local_content? + end + + return if project.nil? + + # We are force-loading the project with the `itself` method + # because the `project` variable can be a `BatchLoader` object and we should not + # pass a `BatchLoader` object in the `for` method to prevent unwanted behaviors. + BatchLoader.for(project.itself) + .batch(key: context.user) do |projects, loader, args| + projects.uniq.each do |project| + context.logger.instrument(:config_file_project_validate_access) do + loader.call(project, Ability.allowed?(args[:key], :download_code, project)) + end + end + end + end + + def sha + return legacy_sha if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + return if project.nil? + + # with `itself`, we are force-loading the project + BatchLoader.for([project.itself, ref_name]) + .batch do |project_ref_pairs, loader| + project_ref_pairs.uniq.each do |project, ref_name| + loader.call([project, ref_name], project.commit(ref_name).try(:sha)) end end end def fetch_local_content - BatchLoader.for([sha, location]) + BatchLoader.for([sha.to_s, location]) .batch(key: project) do |locations, loader, args| context.logger.instrument(:config_file_fetch_project_content) do args[:key].repository.blobs_at(locations).each do |blob| @@ -84,8 +128,22 @@ module Gitlab end end - def sha - strong_memoize(:sha) do + def legacy_project + strong_memoize(:legacy_project) do + ::Project.find_by_full_path(project_name) + end + end + + def legacy_can_access_local_content? + strong_memoize(:legacy_can_access_local_content) do + context.logger.instrument(:config_file_project_validate_access) do + Ability.allowed?(context.user, :download_code, project) + end + end + end + + def legacy_sha + strong_memoize(:legacy_sha) do project.commit(ref_name).try(:sha) end end @@ -94,7 +152,7 @@ module Gitlab def expand_context_attrs { project: project, - sha: sha, + sha: sha.to_s, # we need to use `.to_s` to load the value from the BatchLoader user: context.user, parent_pipeline: context.parent_pipeline, variables: context.variables diff --git a/lib/gitlab/ci/config/external/interpolator.rb b/lib/gitlab/ci/config/external/interpolator.rb new file mode 100644 index 00000000000..f8af77fb246 --- /dev/null +++ b/lib/gitlab/ci/config/external/interpolator.rb @@ -0,0 +1,127 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module External + ## + # Config::External::Interpolation perform includable file interpolation, and surfaces all possible interpolation + # errors. It is designed to provide an external file's validation context too. + # + class Interpolator + include ::Gitlab::Utils::StrongMemoize + + attr_reader :config, :args, :ctx, :errors + + def initialize(config, args, ctx = nil) + @config = config + @args = args.to_h + @ctx = ctx + @errors = [] + + validate! + end + + def valid? + @errors.none? + end + + def ready? + ## + # Interpolation is ready when it has been either interrupted by an error or finished with a result. + # + @result || @errors.any? + end + + def interpolate? + enabled? && has_header? && valid? + end + + def has_header? + config.has_header? && config.header.present? + end + + def to_hash + @result.to_h + end + + def error_message + # Interpolator can have multiple error messages, like: ["interpolation interrupted by errors", "unknown + # interpolation key: `abc`"] ? + # + # We are joining them together into a single one, because only one error can be surfaced when an external + # file gets included and is invalid. The limit to three error messages combined is more than required. + # + @errors.first(3).join(', ') + end + + ## + # TODO Add `instrument.logger` instrumentation blocks: + # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 + # + def interpolate! + return {} unless valid? + return @result ||= content.to_h unless interpolate? + + return @errors.concat(header.errors) unless header.valid? + return @errors.concat(inputs.errors) unless inputs.valid? + return @errors.concat(context.errors) unless context.valid? + return @errors.concat(template.errors) unless template.valid? + + if ctx&.user + ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id) + end + + @result ||= template.interpolated.to_h.deep_symbolize_keys + end + strong_memoize_attr :interpolate! + + private + + def validate! + return errors.push('content does not have a valid YAML syntax') unless config.valid? + + return unless has_header? && !enabled? + + errors.push('can not evaluate included file because interpolation is disabled') + end + + def enabled? + return false if ctx.nil? + + ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) + end + + def header + @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| + header.key = 'header' + + header.compose! + end + end + + def content + @content ||= config.content + end + + def spec + @spec ||= header.inputs_value + end + + def inputs + @inputs ||= Ci::Input::Inputs.new(spec, args) + end + + def context + @context ||= Ci::Interpolation::Context.new({ inputs: inputs.to_hash }) + end + + def template + @template ||= ::Gitlab::Ci::Interpolation::Template + .new(content, context) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/external/mapper/matcher.rb b/lib/gitlab/ci/config/external/mapper/matcher.rb index e59eaa6d324..5072d0971cf 100644 --- a/lib/gitlab/ci/config/external/mapper/matcher.rb +++ b/lib/gitlab/ci/config/external/mapper/matcher.rb @@ -7,22 +7,13 @@ module Gitlab class Mapper # Matches the first file type that matches the given location class Matcher < Base - FILE_CLASSES = [ - External::File::Local, - External::File::Project, - External::File::Component, - External::File::Remote, - External::File::Template, - External::File::Artifact - ].freeze - - FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + include Gitlab::Utils::StrongMemoize private def process_without_instrumentation(locations) locations.map do |location| - matching = FILE_CLASSES.map do |file_class| + matching = file_classes.map do |file_class| file_class.new(location, context) end.select(&:matching?) @@ -31,10 +22,10 @@ module Gitlab elsif matching.empty? raise Mapper::AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. " \ - "Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + "Valid subkeys are: `#{file_subkeys.join('`, `')}`" else raise Mapper::AmbigiousSpecificationError, - "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + "Each include must use only one of: `#{file_subkeys.join('`, `')}`" end end end @@ -42,6 +33,28 @@ module Gitlab def masked_location(location) context.mask_variables_from(location) end + + def file_subkeys + file_classes.map { |f| f.name.demodulize.downcase }.freeze + end + strong_memoize_attr :file_subkeys + + def file_classes + classes = [ + External::File::Local, + External::File::Project, + External::File::Remote, + External::File::Template, + External::File::Artifact + ] + + if Feature.enabled?(:ci_include_components, context.project&.root_namespace) + classes << External::File::Component + end + + classes + end + strong_memoize_attr :file_classes end end end diff --git a/lib/gitlab/ci/config/external/mapper/verifier.rb b/lib/gitlab/ci/config/external/mapper/verifier.rb index 2982b0efb6c..3472f2c581a 100644 --- a/lib/gitlab/ci/config/external/mapper/verifier.rb +++ b/lib/gitlab/ci/config/external/mapper/verifier.rb @@ -9,8 +9,49 @@ module Gitlab class Verifier < Base private + # rubocop: disable Metrics/CyclomaticComplexity def process_without_instrumentation(files) + if ::Feature.disabled?(:ci_batch_project_includes_context, context.project) + return legacy_process_without_instrumentation(files) + end + + files.each do |file| + # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an + # "internal" `include`. We use this condition to exclude that `include` from the included file set. + context.expandset << file unless context.internal_include? + verify_max_includes! + + verify_execution_time! + + file.validate_location! + file.preload_context if file.valid? + end + + # We do not combine the loops because we need to load the context of all files via `BatchLoader`. + files.each do |file| # rubocop:disable Style/CombinableLoops + verify_execution_time! + + file.validate_context! if file.valid? + file.preload_content if file.valid? + end + + # We do not combine the loops because we need to load the content of all files via `BatchLoader`. + files.each do |file| # rubocop:disable Style/CombinableLoops + verify_execution_time! + + file.validate_content! if file.valid? + file.load_and_validate_expanded_hash! if file.valid? + end + end + # rubocop: enable Metrics/CyclomaticComplexity + + def legacy_process_without_instrumentation(files) files.each do |file| + # When running a pipeline, some Ci::ProjectConfig sources prepend the config content with an + # "internal" `include`. We use this condition to exclude that `include` from the included file set. + context.expandset << file unless context.internal_include? + verify_max_includes! + verify_execution_time! file.validate_location! @@ -21,23 +62,15 @@ module Gitlab # We do not combine the loops because we need to load the content of all files before continuing # to call `BatchLoader` for all locations. files.each do |file| # rubocop:disable Style/CombinableLoops - # Checking the max includes will be changed with https://gitlab.com/gitlab-org/gitlab/-/issues/367150 - verify_max_includes! verify_execution_time! file.validate_content! if file.valid? file.load_and_validate_expanded_hash! if file.valid? - - if context.expandset.is_a?(Array) # To be removed when FF 'ci_includes_count_duplicates' is removed - context.expandset << file - else - context.expandset.add(file) - end end end def verify_max_includes! - return if context.expandset.count < context.max_includes + return if context.expandset.count <= context.max_includes raise Mapper::TooManyIncludesError, "Maximum of #{context.max_includes} nested includes are allowed!" end diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb new file mode 100644 index 00000000000..7f0edaaac4c --- /dev/null +++ b/lib/gitlab/ci/config/header/input.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + ## + # Input parameter used for interpolation with the CI configuration. + # + class Input < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + include ::Gitlab::Config::Entry::Attributable + + attributes :default, prefix: :input + + validations do + validates :config, type: Hash, allowed_keys: [:default] + validates :key, alphanumeric: true + validates :input_default, alphanumeric: true, allow_nil: true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/header/root.rb b/lib/gitlab/ci/config/header/root.rb new file mode 100644 index 00000000000..251682d13b4 --- /dev/null +++ b/lib/gitlab/ci/config/header/root.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + ## + # This class represents the root entry of the GitLab CI configuration header. + # + # A header is the first document in a multi-doc YAML that contains metadata + # and specifications about the GitLab CI configuration (the second document). + # + # The header is optional. A CI configuration can also be represented with a + # YAML containing a single document. + class Root < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[spec].freeze + + validations do + validates :config, type: Hash, allowed_keys: ALLOWED_KEYS + end + + entry :spec, Header::Spec, + description: 'Specifications of the CI configuration.', + inherit: false, + default: {} + + def inputs_value + spec_entry.inputs_value + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/header/spec.rb b/lib/gitlab/ci/config/header/spec.rb new file mode 100644 index 00000000000..4753c1eb441 --- /dev/null +++ b/lib/gitlab/ci/config/header/spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Header + class Spec < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + ALLOWED_KEYS = %i[inputs].freeze + + validations do + validates :config, allowed_keys: ALLOWED_KEYS + end + + entry :inputs, ::Gitlab::Config::Entry::ComposableHash, + description: 'Allowed input parameters used for interpolation.', + inherit: false, + metadata: { composable_class: ::Gitlab::Ci::Config::Header::Input } + end + end + end + end +end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index 94ef0afe7f9..729e7e3ac05 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -7,23 +7,39 @@ module Gitlab AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - class << self - def load!(content) + class Loader + def initialize(content, project: nil) + @content = content + @project = project + end + + def load! ensure_custom_tags - if ::Feature.enabled?(:ci_multi_doc_yaml) - Gitlab::Config::Loader::MultiDocYaml.new( + if project.present? && ::Feature.enabled?(:ci_multi_doc_yaml, project) + ::Gitlab::Config::Loader::MultiDocYaml.new( content, max_documents: MAX_DOCUMENTS, - additional_permitted_classes: AVAILABLE_TAGS - ).load!.first + additional_permitted_classes: AVAILABLE_TAGS, + reject_empty: true + ).load! else - Gitlab::Config::Loader::Yaml.new(content, additional_permitted_classes: AVAILABLE_TAGS).load! + ::Gitlab::Config::Loader::Yaml + .new(content, additional_permitted_classes: AVAILABLE_TAGS) + .load! end end + def to_result + Yaml::Result.new(config: load!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e) + end + private + attr_reader :content, :project + def ensure_custom_tags @ensure_custom_tags ||= begin AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } @@ -32,6 +48,23 @@ module Gitlab end end end + + class << self + def load!(content, project: nil) + Loader.new(content, project: project).to_result.then do |result| + ## + # raise an error for backwards compatibility + # + raise result.error unless result.valid? + + result.content + end + end + + def load_result!(content, project: nil) + Loader.new(content, project: project).to_result + end + end end end end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb new file mode 100644 index 00000000000..33f9a454106 --- /dev/null +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Yaml + class Result + attr_reader :error + + def initialize(config: nil, error: nil) + @config = Array.wrap(config) + @error = error + end + + def valid? + error.nil? + end + + def has_header? + return false unless @config.first.is_a?(Hash) + + @config.size > 1 && @config.first.key?(:spec) + end + + def header + raise ArgumentError unless has_header? + + @config.first + end + + def content + return @config.last if has_header? + + @config.first + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/base.rb b/lib/gitlab/ci/input/arguments/base.rb new file mode 100644 index 00000000000..a46037c40ce --- /dev/null +++ b/lib/gitlab/ci/input/arguments/base.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Base is a common abstraction for input arguments: + # - required + # - optional + # - with a default value + # + class Base + attr_reader :key, :value, :spec, :errors + + ArgumentNotValidError = Class.new(StandardError) + + def initialize(key, spec, value) + @key = key # hash key / argument name + @value = value # user-provided value + @spec = spec # configured specification + @errors = [] + + unless value.is_a?(String) || value.nil? # rubocop:disable Style/IfUnlessModifier + @errors.push("unsupported value in input argument `#{key}`") + end + + validate! + end + + def valid? + @errors.none? + end + + def validate! + raise NotImplementedError + end + + def to_value + raise NotImplementedError + end + + def to_hash + raise ArgumentNotValidError unless valid? + + @output ||= { key => to_value } + end + + def self.matches?(spec) + raise NotImplementedError + end + + private + + def error(message) + @errors.push("`#{@key}` input: #{message}") + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/default.rb b/lib/gitlab/ci/input/arguments/default.rb new file mode 100644 index 00000000000..c6762b04870 --- /dev/null +++ b/lib/gitlab/ci/input/arguments/default.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Default class represents user-provided input argument that has a default value. + # + class Default < Input::Arguments::Base + def validate! + return error('argument specification invalid') unless spec.key?(:default) + + error('invalid default value') unless default.is_a?(String) || default.nil? + end + + ## + # User-provided value needs to be specified, but it may be an empty string: + # + # ```yaml + # inputs: + # env: + # default: development + # + # with: + # env: "" + # ``` + # + # The configuration above will result in `env` being an empty string. + # + def to_value + value.nil? ? default : value + end + + def default + spec[:default] + end + + def self.matches?(spec) + return false unless spec.is_a?(Hash) + + spec.count == 1 && spec.each_key.first == :default + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/options.rb b/lib/gitlab/ci/input/arguments/options.rb new file mode 100644 index 00000000000..855dab129be --- /dev/null +++ b/lib/gitlab/ci/input/arguments/options.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Options class represents user-provided input argument that is an enum, and is only valid + # when the value provided is listed as an acceptable one. + # + class Options < Input::Arguments::Base + ## + # An empty value is valid if it is allowlisted: + # + # ```yaml + # inputs: + # run: + # - "" + # - tests + # + # with: + # run: "" + # ``` + # + # The configuration above will return an empty value. + # + def validate! + return error('argument specification invalid') unless options.is_a?(Array) + return error('options argument empty') if options.empty? + + if !value.nil? + error("argument value #{value} not allowlisted") unless options.include?(value) + else + error('argument not provided') + end + end + + def to_value + value + end + + def options + spec[:options] + end + + def self.matches?(spec) + return false unless spec.is_a?(Hash) + + spec.count == 1 && spec.each_key.first == :options + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/required.rb b/lib/gitlab/ci/input/arguments/required.rb new file mode 100644 index 00000000000..2e39f548731 --- /dev/null +++ b/lib/gitlab/ci/input/arguments/required.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Required class represents user-provided required input argument. + # + class Required < Input::Arguments::Base + ## + # The value has to be defined, but it may be empty. + # + def validate! + error('required value has not been provided') if value.nil? + end + + def to_value + value + end + + ## + # Required arguments do not have nested configuration. It has to be defined a null value. + # + # ```yaml + # spec: + # inputs: + # website: + # ``` + # + # An empty string value, that has no specification is also considered as a "required" input, however we should + # never see that being used, because it will be rejected by Ci::Config::Header validation. + # + # ```yaml + # spec: + # inputs: + # website: "" + # ``` + # + # An empty hash value is also considered to be a required argument: + # + # ```yaml + # spec: + # inputs: + # website: {} + # ``` + # + def self.matches?(spec) + spec.blank? + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/arguments/unknown.rb b/lib/gitlab/ci/input/arguments/unknown.rb new file mode 100644 index 00000000000..5873e6e66a6 --- /dev/null +++ b/lib/gitlab/ci/input/arguments/unknown.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + module Arguments + ## + # Input::Arguments::Unknown object gets fabricated when we can't match an input argument entry with any known + # specification. It is matched as the last one, and always returns an error. + # + class Unknown < Input::Arguments::Base + def validate! + if spec.is_a?(Hash) && spec.count == 1 + error("unrecognized input argument specification: `#{spec.each_key.first}`") + else + error('unrecognized input argument definition') + end + end + + def to_value + raise ArgumentError, 'unknown argument value' + end + + def self.matches?(*) + true + end + end + end + end + end +end diff --git a/lib/gitlab/ci/input/inputs.rb b/lib/gitlab/ci/input/inputs.rb new file mode 100644 index 00000000000..1b544e63e7d --- /dev/null +++ b/lib/gitlab/ci/input/inputs.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Input + ## + # Inputs::Input class represents user-provided inputs, configured using `with:` keyword. + # + # Input arguments are only valid with an associated component's inputs specification from component's header. + # + class Inputs + UnknownSpecArgumentError = Class.new(StandardError) + + ARGUMENTS = [ + Input::Arguments::Required, # Input argument is required + Input::Arguments::Default, # Input argument has a default value + Input::Arguments::Options, # Input argument that needs to be allowlisted + Input::Arguments::Unknown # Input argument has not been recognized + ].freeze + + def initialize(spec, args) + @spec = spec.to_h + @args = args.to_h + @inputs = [] + @errors = [] + + validate! + fabricate! + end + + def errors + @errors + @inputs.flat_map(&:errors) + end + + def valid? + errors.none? + end + + def unknown + @args.keys - @spec.keys + end + + def count + @inputs.count + end + + def to_hash + @inputs.inject({}) do |hash, argument| + raise ArgumentError unless argument.valid? + + hash.merge(argument.to_hash) + end + end + + private + + def validate! + @errors.push("unknown input arguments: #{unknown.inspect}") if unknown.any? + end + + def fabricate! + @spec.each do |key, spec| + argument = ARGUMENTS.find { |klass| klass.matches?(spec) } + + raise UnknownSpecArgumentError if argument.nil? + + @inputs.push(argument.new(key, spec, @args[key])) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/interpolation/access.rb b/lib/gitlab/ci/interpolation/access.rb index 42598458902..f9bbd3e118d 100644 --- a/lib/gitlab/ci/interpolation/access.rb +++ b/lib/gitlab/ci/interpolation/access.rb @@ -45,7 +45,11 @@ module Gitlab raise ArgumentError, 'access path invalid' unless valid? @value ||= objects.inject(@ctx) do |memo, value| - memo.fetch(value.to_sym) + key = value.to_sym + + break @errors.push("unknown interpolation key: `#{key}`") unless memo.key?(key) + + memo.fetch(key) end rescue KeyError => e @errors.push(e) diff --git a/lib/gitlab/ci/interpolation/context.rb b/lib/gitlab/ci/interpolation/context.rb index ce7a86a3c9b..69c1fbb792c 100644 --- a/lib/gitlab/ci/interpolation/context.rb +++ b/lib/gitlab/ci/interpolation/context.rb @@ -38,6 +38,10 @@ module Gitlab @context.fetch(field) end + def key?(name) + @context.key?(name) + end + def to_h @context.to_h end @@ -53,7 +57,7 @@ module Gitlab end end - values.max + values.max.to_i end def self.fabricate(context) diff --git a/lib/gitlab/ci/jwt.rb b/lib/gitlab/ci/jwt.rb index d82ca875e76..4ba7b4cc6e1 100644 --- a/lib/gitlab/ci/jwt.rb +++ b/lib/gitlab/ci/jwt.rb @@ -31,6 +31,11 @@ module Gitlab attr_reader :build, :ttl + delegate :project, :user, :pipeline, :runner, to: :build + delegate :source_ref, :source_ref_path, to: :pipeline + delegate :public_key, to: :key + delegate :namespace, to: :project + def reserved_claims now = Time.now.to_i @@ -53,11 +58,12 @@ module Gitlab user_id: user&.id.to_s, user_login: user&.username, user_email: user&.email, - pipeline_id: build.pipeline.id.to_s, - pipeline_source: build.pipeline.source.to_s, + pipeline_id: pipeline.id.to_s, + pipeline_source: pipeline.source.to_s, job_id: build.id.to_s, ref: source_ref, ref_type: ref_type, + ref_path: source_ref_path, ref_protected: build.protected.to_s } @@ -82,30 +88,10 @@ module Gitlab end end - def public_key - key.public_key - end - def kid public_key.to_jwk[:kid] end - def project - build.project - end - - def namespace - project.namespace - end - - def user - build.user - end - - def source_ref - build.pipeline.source_ref - end - def ref_type ::Ci::BuildRunnerPresenter.new(build).ref_type end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index cfefa79d9e0..aff30455d09 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -4,6 +4,8 @@ module Gitlab module Ci class JwtV2 < Jwt DEFAULT_AUD = Settings.gitlab.base_url + GITLAB_HOSTED_RUNNER = 'gitlab-hosted' + SELF_HOSTED_RUNNER = 'self-hosted' def self.for_build(build, aud: DEFAULT_AUD) new(build, ttl: build.metadata_timeout, aud: aud).encoded @@ -20,12 +22,38 @@ module Gitlab attr_reader :aud def reserved_claims - super.merge( + super.merge({ iss: Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: aud + aud: aud, + user_identities: user_identities + }.compact) + end + + def user_identities + return unless user&.pass_user_identities_to_ci_jwt + + user.identities.map do |identity| + { + provider: identity.provider.to_s, + extern_uid: identity.extern_uid.to_s + } + end + end + + def custom_claims + super.merge( + runner_id: runner&.id, + runner_environment: runner_environment, + sha: pipeline.sha ) end + + def runner_environment + return unless runner + + runner.gitlab_hosted? ? GITLAB_HOSTED_RUNNER : SELF_HOSTED_RUNNER + end end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 1b9afc92d6b..447136df81f 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -139,6 +139,7 @@ module Gitlab details: data['details'] || {}, signatures: signatures, project_id: @project.id, + found_by_pipeline: report.pipeline, vulnerability_finding_signatures_enabled: @signatures_enabled)) end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index bef4b147359..92d9d170575 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,27 +7,27 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] + cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], + secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6] }.freeze - VERSIONS_TO_REMOVE_IN_16_0 = %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3].freeze + VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze DEPRECATED_VERSIONS = { - cluster_image_scanning: VERSIONS_TO_REMOVE_IN_16_0, - container_scanning: VERSIONS_TO_REMOVE_IN_16_0, - coverage_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, - dast: VERSIONS_TO_REMOVE_IN_16_0, - api_fuzzing: VERSIONS_TO_REMOVE_IN_16_0, - dependency_scanning: VERSIONS_TO_REMOVE_IN_16_0, - sast: VERSIONS_TO_REMOVE_IN_16_0, - secret_detection: VERSIONS_TO_REMOVE_IN_16_0 + cluster_image_scanning: VERSIONS_TO_REMOVE_IN_17_0, + container_scanning: VERSIONS_TO_REMOVE_IN_17_0, + coverage_fuzzing: VERSIONS_TO_REMOVE_IN_17_0, + dast: VERSIONS_TO_REMOVE_IN_17_0, + api_fuzzing: VERSIONS_TO_REMOVE_IN_17_0, + dependency_scanning: VERSIONS_TO_REMOVE_IN_17_0, + sast: VERSIONS_TO_REMOVE_IN_17_0, + secret_detection: VERSIONS_TO_REMOVE_IN_17_0 }.freeze CURRENT_VERSIONS = SUPPORTED_VERSIONS.to_h { |k, v| [k, v - DEPRECATED_VERSIONS[k]] } @@ -131,11 +131,6 @@ module Gitlab end def report_uses_deprecated_schema_version? - # Avoid deprecation warnings for GitLab security scanners - # To be removed via https://gitlab.com/gitlab-org/gitlab/-/issues/386798 - return if report_data.dig('scan', 'scanner', 'vendor', 'name')&.downcase == 'gitlab' - return if report_data.dig('scan', 'analyzer', 'vendor', 'name')&.downcase == 'gitlab' - DEPRECATED_VERSIONS[report_type].include?(report_version) end diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json deleted file mode 100644 index 14eb376485f..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/container-scanning-report-format.json +++ /dev/null @@ -1,741 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json deleted file mode 100644 index 296a895c7cb..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,711 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json deleted file mode 100644 index 4d3868be019..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dast-report-format.json +++ /dev/null @@ -1,1128 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json deleted file mode 100644 index f0c1a90adcc..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/dependency-scanning-report-format.json +++ /dev/null @@ -1,805 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json deleted file mode 100644 index a7159be0190..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/sast-report-format.json +++ /dev/null @@ -1,706 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json deleted file mode 100644 index 462e23a151c..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.0/secret-detection-report-format.json +++ /dev/null @@ -1,729 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json deleted file mode 100644 index d01e7818866..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/container-scanning-report-format.json +++ /dev/null @@ -1,809 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json deleted file mode 100644 index d496b62ee7f..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,779 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json deleted file mode 100644 index a4d59f39a15..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dast-report-format.json +++ /dev/null @@ -1,1196 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json deleted file mode 100644 index c83d5195be4..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/dependency-scanning-report-format.json +++ /dev/null @@ -1,873 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json deleted file mode 100644 index 7c2cd2b78cf..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/sast-report-format.json +++ /dev/null @@ -1,774 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json deleted file mode 100644 index b4449d0d59c..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.1/secret-detection-report-format.json +++ /dev/null @@ -1,797 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "type": "object", - "description": "The vendor/maintainer of the scanner.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json deleted file mode 100644 index 696fa214abd..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/container-scanning-report-format.json +++ /dev/null @@ -1,871 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json deleted file mode 100644 index 1312696d642..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,841 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json deleted file mode 100644 index a7e9f83e557..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dast-report-format.json +++ /dev/null @@ -1,1258 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json deleted file mode 100644 index d6ff5248358..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/dependency-scanning-report-format.json +++ /dev/null @@ -1,935 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json deleted file mode 100644 index 2be6801d2f6..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/sast-report-format.json +++ /dev/null @@ -1,836 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json deleted file mode 100644 index c44554489ce..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.2/secret-detection-report-format.json +++ /dev/null @@ -1,859 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json deleted file mode 100644 index 959b7b8f6f2..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/container-scanning-report-format.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json deleted file mode 100644 index 20038dcb21c..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json deleted file mode 100644 index 37b98a73233..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dast-report-format.json +++ /dev/null @@ -1,1291 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json deleted file mode 100644 index 5e9bbeec1a9..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json deleted file mode 100644 index 8aa98646818..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json deleted file mode 100644 index 5a315e39385..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.3/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json deleted file mode 100644 index 3736eac0ba0..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json deleted file mode 100644 index e324201b04b..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/container-scanning-report-format.json +++ /dev/null @@ -1,904 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json deleted file mode 100644 index 7ac5d2b7783..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json deleted file mode 100644 index b3ce7609aea..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dast-report-format.json +++ /dev/null @@ -1,1291 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json deleted file mode 100644 index 605d379e497..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json deleted file mode 100644 index 2d9e1af6663..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json deleted file mode 100644 index 70f22b243c6..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.4/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.4" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json deleted file mode 100644 index 882a21e430a..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json deleted file mode 100644 index 08f38650340..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/container-scanning-report-format.json +++ /dev/null @@ -1,910 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image." - }, - "default_branch_image": { - "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+:[a-zA-Z0-9_.-]+$", - "description": "The name of the image on the default branch." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json deleted file mode 100644 index a442d38c134..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json deleted file mode 100644 index 9a4d1515bc2..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dast-report-format.json +++ /dev/null @@ -1,1291 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json deleted file mode 100644 index e84dd9c87d8..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json deleted file mode 100644 index b10b199a97c..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json deleted file mode 100644 index 5bd945c8ab5..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.5/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.5" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability.", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json deleted file mode 100644 index 951b0fea013..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.6" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json deleted file mode 100644 index de79d4b52ab..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.6" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json deleted file mode 100644 index 80d6fc9c7d2..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.6" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json deleted file mode 100644 index b87182bb237..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.6" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json deleted file mode 100644 index 191d94aad5f..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.0.6" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json deleted file mode 100644 index 3f78ff0354f..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json deleted file mode 100644 index 6e8a1c54fb4..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/container-scanning-report-format.json +++ /dev/null @@ -1,911 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", - "description": "The analyzed Docker image." - }, - "default_branch_image": { - "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", - "description": "The name of the image on the default branch." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json deleted file mode 100644 index 73c03082d32..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dast-report-format.json +++ /dev/null @@ -1,1291 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.0" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json deleted file mode 100644 index a13e0418499..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/container-scanning-report-format.json +++ /dev/null @@ -1,911 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", - "description": "The analyzed Docker image." - }, - "default_branch_image": { - "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", - "description": "The name of the image on the default branch." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json deleted file mode 100644 index 050c34669b3..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json deleted file mode 100644 index 62ed293ad44..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dast-report-format.json +++ /dev/null @@ -1,1291 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "minLength": 1, - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json deleted file mode 100644 index 1e3f4188845..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json deleted file mode 100644 index 4c57d20dbaa..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json deleted file mode 100644 index b1337954e97..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.1" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json deleted file mode 100644 index 31840a7e914..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json deleted file mode 100644 index c70628a0949..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/container-scanning-report-format.json +++ /dev/null @@ -1,911 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+$", - "description": "The analyzed Docker image." - }, - "default_branch_image": { - "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", - "description": "The name of the image on the default branch." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json deleted file mode 100644 index fbc7b4ea733..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json deleted file mode 100644 index 3c9db0546b1..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dast-report-format.json +++ /dev/null @@ -1,1287 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json deleted file mode 100644 index c7459216faf..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json deleted file mode 100644 index 20818792652..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json deleted file mode 100644 index 12386d2c1d4..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.2/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.2" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json deleted file mode 100644 index db4c7ab1425..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json +++ /dev/null @@ -1,977 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Cluster Image Scanning", - "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "cluster_image_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "image", - "kubernetes_resource" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "description": "The analyzed Docker image.", - "examples": [ - "index.docker.io/library/nginx:1.21" - ] - }, - "kubernetes_resource": { - "type": "object", - "description": "The specific Kubernetes resource that was scanned.", - "required": [ - "namespace", - "kind", - "name", - "container_name" - ], - "properties": { - "namespace": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes namespace the resource that had its image scanned.", - "examples": [ - "default", - "staging", - "production" - ] - }, - "kind": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The Kubernetes kind the resource that had its image scanned.", - "examples": [ - "Deployment", - "DaemonSet" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the resource that had its image scanned.", - "examples": [ - "nginx-ingress" - ] - }, - "container_name": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The name of the container that had its image scanned.", - "examples": [ - "nginx" - ] - }, - "agent_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", - "examples": [ - "1234" - ] - }, - "cluster_id": { - "type": "string", - "minLength": 1, - "maxLength": 255, - "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", - "examples": [ - "1234" - ] - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json deleted file mode 100644 index 641cfc82e48..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json +++ /dev/null @@ -1,911 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Container Scanning", - "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "container_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "dependency", - "operating_system", - "image" - ], - "properties": { - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - }, - "operating_system": { - "type": "string", - "minLength": 1, - "description": "The operating system that contains the vulnerable package." - }, - "image": { - "type": "string", - "minLength": 1, - "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", - "description": "The analyzed Docker image." - }, - "default_branch_image": { - "type": "string", - "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", - "description": "The name of the image on the default branch." - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json deleted file mode 100644 index 59aa172444d..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json +++ /dev/null @@ -1,874 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Fuzz Testing", - "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "coverage_fuzzing" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "description": "The location of the error", - "type": "object", - "properties": { - "crash_address": { - "type": "string", - "description": "The relative address in memory were the crash occurred.", - "examples": [ - "0xabababab" - ] - }, - "stacktrace_snippet": { - "type": "string", - "description": "The stack trace recorded during fuzzing resulting the crash.", - "examples": [ - "func_a+0xabcd\nfunc_b+0xabcc" - ] - }, - "crash_state": { - "type": "string", - "description": "Minimised and normalized crash stack-trace (called crash_state).", - "examples": [ - "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" - ] - }, - "crash_type": { - "type": "string", - "description": "Type of the crash.", - "examples": [ - "Heap-Buffer-overflow", - "Division-by-zero" - ] - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json deleted file mode 100644 index 0e4c866794a..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json +++ /dev/null @@ -1,1287 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab DAST", - "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanned_resources", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dast", - "api_fuzzing" - ] - }, - "scanned_resources": { - "type": "array", - "description": "The attack surface scanned by DAST.", - "items": { - "type": "object", - "required": [ - "method", - "url", - "type" - ], - "properties": { - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method of the scanned resource.", - "examples": [ - "GET", - "POST", - "HEAD" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the scanned resource.", - "examples": [ - "http://my.site.com/a-page" - ] - }, - "type": { - "type": "string", - "minLength": 1, - "description": "Type of the scanned resource, for DAST, this must be 'url'.", - "examples": [ - "url" - ] - } - } - } - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "evidence": { - "type": "object", - "properties": { - "source": { - "type": "object", - "description": "Source of evidence", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "Unique source identifier", - "examples": [ - "assert:LogAnalysis", - "assert:StatusCode" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Source display name", - "examples": [ - "Log Analysis", - "Status Code" - ] - }, - "url": { - "type": "string", - "description": "Link to additional information", - "examples": [ - "https://docs.gitlab.com/ee/development/integrations/secure.html" - ] - } - } - }, - "summary": { - "type": "string", - "description": "Human readable string containing evidence of the vulnerability.", - "examples": [ - "Credit card 4111111111111111 found", - "Server leaked information nginx/1.17.6" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - }, - "supporting_messages": { - "type": "array", - "description": "Array of supporting http messages.", - "items": { - "type": "object", - "description": "A supporting http message.", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Message display name.", - "examples": [ - "Unmodified", - "Recorded" - ] - }, - "request": { - "type": "object", - "description": "An HTTP request.", - "required": [ - "headers", - "method", - "url" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "method": { - "type": "string", - "minLength": 1, - "description": "HTTP method used in the request.", - "examples": [ - "GET", - "POST" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "URL of the request.", - "examples": [ - "http://my.site.com/vulnerable-endpoint?show-credit-card" - ] - }, - "body": { - "type": "string", - "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "user=jsmith&first=%27&last=smith" - ] - } - } - }, - "response": { - "type": "object", - "description": "An HTTP response.", - "required": [ - "headers", - "reason_phrase", - "status_code" - ], - "properties": { - "headers": { - "type": "array", - "description": "HTTP headers present on the request.", - "items": { - "type": "object", - "required": [ - "name", - "value" - ], - "properties": { - "name": { - "type": "string", - "minLength": 1, - "description": "Name of the HTTP header.", - "examples": [ - "Accept", - "Content-Length", - "Content-Type" - ] - }, - "value": { - "type": "string", - "description": "Value of the HTTP header.", - "examples": [ - "*/*", - "560", - "application/json; charset=utf-8" - ] - } - } - } - }, - "reason_phrase": { - "type": "string", - "description": "HTTP reason phrase of the response.", - "examples": [ - "OK", - "Internal Server Error" - ] - }, - "status_code": { - "type": "integer", - "description": "HTTP status code of the response.", - "examples": [ - 200, - 500 - ] - }, - "body": { - "type": "string", - "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", - "examples": [ - "{\"user_id\": 2}" - ] - } - } - } - } - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "hostname": { - "type": "string", - "description": "The protocol, domain, and port of the application where the vulnerability was found." - }, - "method": { - "type": "string", - "description": "The HTTP method that was used to request the URL where the vulnerability was found." - }, - "param": { - "type": "string", - "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." - }, - "path": { - "type": "string", - "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." - } - } - }, - "assets": { - "type": "array", - "description": "Array of build assets associated with vulnerability.", - "items": { - "type": "object", - "description": "Describes an asset associated with vulnerability.", - "required": [ - "type", - "name", - "url" - ], - "properties": { - "type": { - "type": "string", - "description": "The type of asset", - "enum": [ - "http_session", - "postman" - ] - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Display name for asset", - "examples": [ - "HTTP Messages", - "Postman Collection" - ] - }, - "url": { - "type": "string", - "minLength": 1, - "description": "Link to asset in build artifacts", - "examples": [ - "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" - ] - } - } - } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json deleted file mode 100644 index 652c2f48fe4..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json +++ /dev/null @@ -1,968 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Dependency Scanning", - "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "dependency_files", - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "dependency_scanning" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "required": [ - "file", - "dependency" - ], - "properties": { - "file": { - "type": "string", - "minLength": 1, - "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." - }, - "dependency": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - }, - "dependency_files": { - "type": "array", - "description": "List of dependency files identified in the project.", - "items": { - "type": "object", - "required": [ - "path", - "package_manager", - "dependencies" - ], - "properties": { - "path": { - "type": "string", - "minLength": 1 - }, - "package_manager": { - "type": "string", - "minLength": 1 - }, - "dependencies": { - "type": "array", - "items": { - "type": "object", - "description": "Describes the dependency of a project where the vulnerability is located.", - "properties": { - "package": { - "type": "object", - "description": "Provides information on the package where the vulnerability is located.", - "properties": { - "name": { - "type": "string", - "description": "Name of the package where the vulnerability is located." - } - } - }, - "version": { - "type": "string", - "description": "Version of the vulnerable package." - }, - "iid": { - "description": "ID that identifies the dependency in the scope of a dependency file.", - "type": "number" - }, - "direct": { - "type": "boolean", - "description": "Tells whether this is a direct, top-level dependency of the scanned project." - }, - "dependency_path": { - "type": "array", - "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", - "items": { - "type": "object", - "required": [ - "iid" - ], - "properties": { - "iid": { - "type": "number", - "description": "ID that is unique in the scope of a parent object, and specific to the resource type." - } - } - } - } - } - } - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json deleted file mode 100644 index 40d4d9f5287..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json +++ /dev/null @@ -1,869 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab SAST", - "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "sast" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "type": "object", - "description": "Identifies the vulnerability's location.", - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability." - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located." - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located." - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json deleted file mode 100644 index cfde126dd7b..00000000000 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json +++ /dev/null @@ -1,892 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Report format for GitLab Secret Detection", - "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", - "definitions": { - "detail_type": { - "oneOf": [ - { - "$ref": "#/definitions/named_list" - }, - { - "$ref": "#/definitions/list" - }, - { - "$ref": "#/definitions/table" - }, - { - "$ref": "#/definitions/text" - }, - { - "$ref": "#/definitions/url" - }, - { - "$ref": "#/definitions/code" - }, - { - "$ref": "#/definitions/value" - }, - { - "$ref": "#/definitions/diff" - }, - { - "$ref": "#/definitions/markdown" - }, - { - "$ref": "#/definitions/commit" - }, - { - "$ref": "#/definitions/file_location" - }, - { - "$ref": "#/definitions/module_location" - } - ] - }, - "text_value": { - "type": "string" - }, - "named_field": { - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "$ref": "#/definitions/text_value", - "minLength": 1 - }, - "description": { - "$ref": "#/definitions/text_value" - } - } - }, - "named_list": { - "type": "object", - "description": "An object with named and typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "named-list" - }, - "items": { - "type": "object", - "patternProperties": { - "^.*$": { - "allOf": [ - { - "$ref": "#/definitions/named_field" - }, - { - "$ref": "#/definitions/detail_type" - } - ] - } - } - } - } - }, - "list": { - "type": "object", - "description": "A list of typed fields", - "required": [ - "type", - "items" - ], - "properties": { - "type": { - "const": "list" - }, - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - }, - "table": { - "type": "object", - "description": "A table of typed fields", - "required": [ - "type", - "rows" - ], - "properties": { - "type": { - "const": "table" - }, - "header": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - }, - "rows": { - "type": "array", - "items": { - "type": "array", - "items": { - "$ref": "#/definitions/detail_type" - } - } - } - } - }, - "text": { - "type": "object", - "description": "Raw text", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "text" - }, - "value": { - "$ref": "#/definitions/text_value" - } - } - }, - "url": { - "type": "object", - "description": "A single URL", - "required": [ - "type", - "href" - ], - "properties": { - "type": { - "const": "url" - }, - "text": { - "$ref": "#/definitions/text_value" - }, - "href": { - "type": "string", - "minLength": 1, - "examples": [ - "http://mysite.com" - ] - } - } - }, - "code": { - "type": "object", - "description": "A codeblock", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "code" - }, - "value": { - "type": "string" - }, - "lang": { - "type": "string", - "description": "A programming language" - } - } - }, - "value": { - "type": "object", - "description": "A field that can store a range of types of value", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "value" - }, - "value": { - "type": [ - "number", - "string", - "boolean" - ] - } - } - }, - "diff": { - "type": "object", - "description": "A diff", - "required": [ - "type", - "before", - "after" - ], - "properties": { - "type": { - "const": "diff" - }, - "before": { - "type": "string" - }, - "after": { - "type": "string" - } - } - }, - "markdown": { - "type": "object", - "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "markdown" - }, - "value": { - "$ref": "#/definitions/text_value", - "examples": [ - "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" - ] - } - } - }, - "commit": { - "type": "object", - "description": "A commit/tag/branch within the GitLab project", - "required": [ - "type", - "value" - ], - "properties": { - "type": { - "const": "commit" - }, - "value": { - "type": "string", - "description": "The commit SHA", - "minLength": 1 - } - } - }, - "file_location": { - "type": "object", - "description": "A location within a file in the project", - "required": [ - "type", - "file_name", - "line_start" - ], - "properties": { - "type": { - "const": "file-location" - }, - "file_name": { - "type": "string", - "minLength": 1 - }, - "line_start": { - "type": "integer" - }, - "line_end": { - "type": "integer" - } - } - }, - "module_location": { - "type": "object", - "description": "A location within a binary module of the form module+relative_offset", - "required": [ - "type", - "module_name", - "offset" - ], - "properties": { - "type": { - "const": "module-location" - }, - "module_name": { - "type": "string", - "minLength": 1, - "examples": [ - "compiled_binary" - ] - }, - "offset": { - "type": "integer", - "examples": [ - 100 - ] - } - } - } - }, - "self": { - "version": "14.1.3" - }, - "required": [ - "version", - "vulnerabilities" - ], - "additionalProperties": true, - "properties": { - "scan": { - "type": "object", - "required": [ - "end_time", - "scanner", - "start_time", - "status", - "type" - ], - "properties": { - "end_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-01-28T03:26:02" - ] - }, - "messages": { - "type": "array", - "items": { - "type": "object", - "description": "Communication intended for the initiator of a scan.", - "required": [ - "level", - "value" - ], - "properties": { - "level": { - "type": "string", - "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", - "enum": [ - "info", - "warn", - "fatal" - ], - "examples": [ - "info" - ] - }, - "value": { - "type": "string", - "description": "The message to communicate.", - "minLength": 1, - "examples": [ - "Permission denied, scanning aborted" - ] - } - } - } - }, - "analyzer": { - "type": "object", - "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the analyzer.", - "minLength": 1, - "examples": [ - "gitlab-dast" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the analyzer, not required to be unique.", - "minLength": 1, - "examples": [ - "GitLab DAST" - ] - }, - "url": { - "type": "string", - "format": "uri", - "pattern": "^https?://.+", - "description": "A link to more information about the analyzer.", - "examples": [ - "https://docs.gitlab.com/ee/user/application_security/dast" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the analyzer.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - }, - "version": { - "type": "string", - "description": "The version of the analyzer.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - } - } - }, - "scanner": { - "type": "object", - "description": "Object defining the scanner used to perform the scan.", - "required": [ - "id", - "name", - "version", - "vendor" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique id that identifies the scanner.", - "minLength": 1, - "examples": [ - "my-sast-scanner" - ] - }, - "name": { - "type": "string", - "description": "A human readable value that identifies the scanner, not required to be unique.", - "minLength": 1, - "examples": [ - "My SAST Scanner" - ] - }, - "url": { - "type": "string", - "description": "A link to more information about the scanner.", - "examples": [ - "https://scanner.url" - ] - }, - "version": { - "type": "string", - "description": "The version of the scanner.", - "minLength": 1, - "examples": [ - "1.0.2" - ] - }, - "vendor": { - "description": "The vendor/maintainer of the scanner.", - "type": "object", - "required": [ - "name" - ], - "properties": { - "name": { - "type": "string", - "description": "The name of the vendor.", - "minLength": 1, - "examples": [ - "GitLab" - ] - } - } - } - } - }, - "start_time": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", - "examples": [ - "2020-02-14T16:01:59" - ] - }, - "status": { - "type": "string", - "description": "Result of the scan.", - "enum": [ - "success", - "failure" - ] - }, - "type": { - "type": "string", - "description": "Type of the scan.", - "enum": [ - "secret_detection" - ] - } - } - }, - "schema": { - "type": "string", - "description": "URI pointing to the validating security report schema.", - "format": "uri" - }, - "version": { - "type": "string", - "description": "The version of the schema to which the JSON report conforms.", - "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" - }, - "vulnerabilities": { - "type": "array", - "description": "Array of vulnerability objects.", - "items": { - "type": "object", - "description": "Describes the vulnerability using GitLab Flavored Markdown", - "required": [ - "category", - "cve", - "identifiers", - "location", - "scanner" - ], - "properties": { - "id": { - "type": "string", - "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", - "examples": [ - "642735a5-1425-428d-8d4e-3c854885a3c9" - ] - }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, - "name": { - "type": "string", - "description": "The name of the vulnerability. This must not include the finding's specific information." - }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, - "description": { - "type": "string", - "description": "A long text section describing the vulnerability more fully." - }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, - "severity": { - "type": "string", - "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", - "enum": [ - "Info", - "Unknown", - "Low", - "Medium", - "High", - "Critical" - ] - }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, - "solution": { - "type": "string", - "description": "Explanation of how to fix the vulnerability." - }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, - "identifiers": { - "type": "array", - "minItems": 1, - "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", - "items": { - "type": "object", - "required": [ - "type", - "name", - "value" - ], - "properties": { - "type": { - "type": "string", - "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", - "minLength": 1 - }, - "name": { - "type": "string", - "description": "Human-readable name of the identifier.", - "minLength": 1 - }, - "url": { - "type": "string", - "description": "URL of the identifier's documentation.", - "format": "uri" - }, - "value": { - "type": "string", - "description": "Value of the identifier, for matching purpose.", - "minLength": 1 - } - } - } - }, - "links": { - "type": "array", - "description": "An array of references to external documentation or articles that describe the vulnerability.", - "items": { - "type": "object", - "required": [ - "url" - ], - "properties": { - "name": { - "type": "string", - "description": "Name of the vulnerability details link." - }, - "url": { - "type": "string", - "description": "URL of the vulnerability details document.", - "format": "uri" - } - } - } - }, - "details": { - "$ref": "#/definitions/named_list/properties/items" - }, - "tracking": { - "description": "Describes how this vulnerability should be tracked as the project changes.", - "oneOf": [ - { - "description": "Declares that a series of items should be tracked using source-specific tracking methods.", - "required": [ - "items" - ], - "properties": { - "type": { - "const": "source" - }, - "items": { - "type": "array", - "items": { - "description": "An item that should be tracked using source-specific tracking methods.", - "type": "object", - "required": [ - "signatures" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located." - }, - "start_line": { - "type": "number", - "description": "The first line of the file that includes the vulnerability." - }, - "end_line": { - "type": "number", - "description": "The last line of the file that includes the vulnerability." - }, - "signatures": { - "type": "array", - "description": "An array of calculated tracking signatures for this tracking item.", - "minItems": 1, - "items": { - "description": "A calculated tracking signature value and metadata.", - "required": [ - "algorithm", - "value" - ], - "properties": { - "algorithm": { - "type": "string", - "description": "The algorithm used to generate the signature." - }, - "value": { - "type": "string", - "description": "The result of this signature algorithm." - } - } - } - } - } - } - } - } - } - ], - "properties": { - "type": { - "type": "string", - "description": "Each tracking type must declare its own type." - } - } - }, - "flags": { - "description": "Flags that can be attached to vulnerabilities.", - "type": "array", - "items": { - "type": "object", - "description": "Informational flags identified and assigned to a vulnerability.", - "required": [ - "type", - "origin", - "description" - ], - "properties": { - "type": { - "type": "string", - "minLength": 1, - "description": "Result of the scan.", - "enum": [ - "flagged-as-likely-false-positive" - ] - }, - "origin": { - "minLength": 1, - "description": "Tool that issued the flag.", - "type": "string" - }, - "description": { - "minLength": 1, - "description": "What the flag is about.", - "type": "string" - } - } - } - }, - "location": { - "required": [ - "commit" - ], - "properties": { - "file": { - "type": "string", - "description": "Path to the file where the vulnerability is located" - }, - "commit": { - "type": "object", - "description": "Represents the commit in which the vulnerability was detected", - "required": [ - "sha" - ], - "properties": { - "author": { - "type": "string" - }, - "date": { - "type": "string" - }, - "message": { - "type": "string" - }, - "sha": { - "type": "string", - "minLength": 1 - } - } - }, - "start_line": { - "type": "number", - "description": "The first line of the code affected by the vulnerability" - }, - "end_line": { - "type": "number", - "description": "The last line of the code affected by the vulnerability" - }, - "class": { - "type": "string", - "description": "Provides the name of the class where the vulnerability is located" - }, - "method": { - "type": "string", - "description": "Provides the name of the method where the vulnerability is located" - } - } - }, - "raw_source_code_extract": { - "type": "string", - "description": "Provides an unsanitized excerpt of the affected source code." - } - } - } - }, - "remediations": { - "type": "array", - "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", - "items": { - "type": "object", - "required": [ - "fixes", - "summary", - "diff" - ], - "properties": { - "fixes": { - "type": "array", - "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", - "items": { - "type": "object", - "required": [ - "cve" - ], - "properties": { - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - } - } - } - }, - "summary": { - "type": "string", - "minLength": 1, - "description": "An overview of how the vulnerabilities were fixed." - }, - "diff": { - "type": "string", - "minLength": 1, - "description": "A base64-encoded remediation code diff, compatible with git apply." - } - } - } - } - } -} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json index 7bcb2d5867f..91414255211 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.1/cluster-image-scanning-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/cluster-image-scanning-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", "title": "Report format for GitLab Cluster Image Scanning", "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.1.1" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -346,7 +351,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -384,6 +389,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -412,7 +468,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -509,7 +564,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -528,13 +583,47 @@ "enum": [ "cluster_image_scanning" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -548,41 +637,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -595,43 +672,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -657,7 +702,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -683,7 +728,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -692,6 +737,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -730,6 +776,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -804,10 +851,17 @@ "dependency": { "type": "object", "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], "properties": { "package": { "type": "object", "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], "properties": { "name": { "type": "string", @@ -950,12 +1004,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json index fb412af44e3..ecd92ed2ff1 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/container-scanning-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/container-scanning-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", "title": "Report format for GitLab Container Scanning", "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.0.6" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -346,7 +351,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -384,6 +389,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -412,7 +468,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -509,7 +564,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -528,13 +583,47 @@ "enum": [ "container_scanning" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -548,41 +637,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -595,43 +672,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -657,7 +702,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -683,7 +728,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -692,6 +737,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -730,6 +776,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -804,10 +851,17 @@ "dependency": { "type": "object", "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], "properties": { "package": { "type": "object", "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], "properties": { "name": { "type": "string", @@ -858,7 +912,6 @@ "default_branch_image": { "type": "string", "maxLength": 255, - "pattern": "^[a-zA-Z0-9/_.-]+:[a-zA-Z0-9_.-]+$", "description": "The name of the image on the default branch." } } @@ -883,12 +936,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json index f63ebfa2cc2..11a1375710b 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/coverage-fuzzing-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/coverage-fuzzing-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", "title": "Report format for GitLab Fuzz Testing", "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.1.0" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -346,7 +351,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -384,6 +389,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -412,7 +468,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -509,7 +564,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -528,13 +583,47 @@ "enum": [ "coverage_fuzzing" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -548,41 +637,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -595,43 +672,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -657,7 +702,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -683,7 +728,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -692,6 +737,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -730,6 +776,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -847,12 +894,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json index 598f162aad2..1351cb261e0 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.0.6/dast-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dast-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", "title": "Report format for GitLab DAST", "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.0.6" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanned_resources", "scanner", @@ -347,7 +352,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -385,6 +390,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -413,7 +469,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -510,7 +565,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -531,6 +586,40 @@ "api_fuzzing" ] }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, "scanned_resources": { "type": "array", "description": "The attack surface scanned by DAST.", @@ -576,7 +665,7 @@ "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -590,41 +679,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -637,43 +714,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -699,7 +744,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -725,7 +770,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -734,6 +779,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -772,6 +818,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -911,7 +958,6 @@ }, "value": { "type": "string", - "minLength": 1, "description": "Value of the HTTP header.", "examples": [ "*/*", @@ -979,7 +1025,6 @@ }, "value": { "type": "string", - "minLength": 1, "description": "Value of the HTTP header.", "examples": [ "*/*", @@ -1065,7 +1110,6 @@ }, "value": { "type": "string", - "minLength": 1, "description": "Value of the HTTP header.", "examples": [ "*/*", @@ -1133,7 +1177,6 @@ }, "value": { "type": "string", - "minLength": 1, "description": "Value of the HTTP header.", "examples": [ "*/*", @@ -1235,14 +1278,6 @@ } } } - }, - "discovered_at": { - "type": "string", - "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", - "examples": [ - "2020-01-28T03:26:02.956" - ] } } } @@ -1264,12 +1299,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json index 6f2c3740b09..e4b02362cb1 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/dependency-scanning-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/dependency-scanning-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", "title": "Report format for GitLab Dependency Scanning", "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,10 +327,12 @@ } }, "self": { - "version": "14.1.0" + "version": "15.0.6" }, + "type": "object", "required": [ "dependency_files", + "scan", "version", "vulnerabilities" ], @@ -337,6 +341,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -347,7 +352,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -385,6 +390,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -413,7 +469,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -510,7 +565,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -529,13 +584,47 @@ "enum": [ "dependency_scanning" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -549,41 +638,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -596,43 +673,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -658,7 +703,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -684,7 +729,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -693,6 +738,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -731,6 +777,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -809,10 +856,17 @@ "dependency": { "type": "object", "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], "properties": { "package": { "type": "object", "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], "properties": { "name": { "type": "string", @@ -872,12 +926,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } @@ -919,10 +977,17 @@ "items": { "type": "object", "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], "properties": { "package": { "type": "object", "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], "properties": { "name": { "type": "string", diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json index 5c7f636e169..e4cb5fb2985 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/sast-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/sast-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", "title": "Report format for GitLab SAST", "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.1.0" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -346,7 +351,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -384,6 +389,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -412,7 +468,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -509,7 +564,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -528,13 +583,47 @@ "enum": [ "sast" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -548,41 +637,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -595,43 +672,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -657,7 +702,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -683,7 +728,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -692,6 +737,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -730,6 +776,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -842,12 +889,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json index a87388c45e7..5eb52b11efe 100644 --- a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.0/secret-detection-report-format.json +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.6/secret-detection-report-format.json @@ -1,5 +1,6 @@ { "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", "title": "Report format for GitLab Secret Detection", "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", "definitions": { @@ -54,6 +55,7 @@ "properties": { "name": { "$ref": "#/definitions/text_value", + "type": "string", "minLength": 1 }, "description": { @@ -325,9 +327,11 @@ } }, "self": { - "version": "14.1.0" + "version": "15.0.6" }, + "type": "object", "required": [ + "scan", "version", "vulnerabilities" ], @@ -336,6 +340,7 @@ "scan": { "type": "object", "required": [ + "analyzer", "end_time", "scanner", "start_time", @@ -346,7 +351,7 @@ "end_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-01-28T03:26:02" ] @@ -384,6 +389,57 @@ } } }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, "analyzer": { "type": "object", "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", @@ -412,7 +468,6 @@ }, "url": { "type": "string", - "format": "uri", "pattern": "^https?://.+", "description": "A link to more information about the analyzer.", "examples": [ @@ -509,7 +564,7 @@ "start_time": { "type": "string", "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", - "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", "examples": [ "2020-02-14T16:01:59" ] @@ -528,13 +583,47 @@ "enum": [ "secret_detection" ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } } } }, "schema": { "type": "string", "description": "URI pointing to the validating security report schema.", - "format": "uri" + "pattern": "^https?://.+" }, "version": { "type": "string", @@ -548,41 +637,29 @@ "type": "object", "description": "Describes the vulnerability using GitLab Flavored Markdown", "required": [ - "category", - "cve", + "id", "identifiers", - "location", - "scanner" + "location" ], "properties": { "id": { "type": "string", + "minLength": 1, "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", "examples": [ "642735a5-1425-428d-8d4e-3c854885a3c9" ] }, - "category": { - "type": "string", - "minLength": 1, - "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." - }, "name": { "type": "string", + "maxLength": 255, "description": "The name of the vulnerability. This must not include the finding's specific information." }, - "message": { - "type": "string", - "description": "A short text section that describes the vulnerability. This may include the finding's specific information." - }, "description": { "type": "string", + "maxLength": 1048576, "description": "A long text section describing the vulnerability more fully." }, - "cve": { - "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." - }, "severity": { "type": "string", "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", @@ -595,43 +672,11 @@ "Critical" ] }, - "confidence": { - "type": "string", - "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", - "enum": [ - "Ignore", - "Unknown", - "Experimental", - "Low", - "Medium", - "High", - "Confirmed" - ] - }, "solution": { "type": "string", + "maxLength": 7000, "description": "Explanation of how to fix the vulnerability." }, - "scanner": { - "description": "Describes the scanner used to find this vulnerability.", - "type": "object", - "required": [ - "id", - "name" - ], - "properties": { - "id": { - "type": "string", - "minLength": 1, - "description": "The scanner's ID, as a snake_case string." - }, - "name": { - "type": "string", - "minLength": 1, - "description": "Human-readable name of the scanner." - } - } - }, "identifiers": { "type": "array", "minItems": 1, @@ -657,7 +702,7 @@ "url": { "type": "string", "description": "URL of the identifier's documentation.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" }, "value": { "type": "string", @@ -683,7 +728,7 @@ "url": { "type": "string", "description": "URL of the vulnerability details document.", - "format": "uri" + "pattern": "^(https?|ftp)://.+" } } } @@ -692,6 +737,7 @@ "$ref": "#/definitions/named_list/properties/items" }, "tracking": { + "type": "object", "description": "Describes how this vulnerability should be tracked as the project changes.", "oneOf": [ { @@ -730,6 +776,7 @@ "minItems": 1, "items": { "description": "A calculated tracking signature value and metadata.", + "type": "object", "required": [ "algorithm", "value" @@ -796,6 +843,7 @@ "required": [ "commit" ], + "type": "object", "properties": { "file": { "type": "string", @@ -865,12 +913,16 @@ "items": { "type": "object", "required": [ - "cve" + "id" ], "properties": { - "cve": { + "id": { "type": "string", - "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] } } } diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index d2dc712e366..4bc2f6c7be7 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -13,7 +13,8 @@ module Gitlab :seeds_block, :variables_attributes, :push_options, :chat_data, :allow_mirror_update, :bridge, :content, :dry_run, :logger, # These attributes are set by Chains during processing: - :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed + :config_content, :yaml_processor_result, :workflow_rules_result, :pipeline_seed, + :pipeline_config ) do include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index d41213ef6dd..779aac7d520 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -14,6 +14,7 @@ module Gitlab @pipeline.build_pipeline_config(content: pipeline_config.content) @command.config_content = pipeline_config.content @pipeline.config_source = pipeline_config.source + @command.pipeline_config = pipeline_config else error('Missing CI config file') end diff --git a/lib/gitlab/ci/pipeline/chain/config/process.rb b/lib/gitlab/ci/pipeline/chain/config/process.rb index ad6b2fd3411..4976e075727 100644 --- a/lib/gitlab/ci/pipeline/chain/config/process.rb +++ b/lib/gitlab/ci/pipeline/chain/config/process.rb @@ -20,6 +20,7 @@ module Gitlab source: @pipeline.source, user: current_user, parent_pipeline: parent_pipeline, + pipeline_config: @command.pipeline_config, logger: logger } ) diff --git a/lib/gitlab/ci/pipeline/chain/limit/activity.rb b/lib/gitlab/ci/pipeline/chain/limit/activity.rb deleted file mode 100644 index ef9235477db..00000000000 --- a/lib/gitlab/ci/pipeline/chain/limit/activity.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Chain - module Limit - class Activity < Chain::Base - def perform! - # to be overridden in EE - end - - def break? - false # to be overridden in EE - end - end - end - end - end - end -end - -Gitlab::Ci::Pipeline::Chain::Limit::Activity.prepend_mod_with('Gitlab::Ci::Pipeline::Chain::Limit::Activity') diff --git a/lib/gitlab/ci/pipeline/duration.rb b/lib/gitlab/ci/pipeline/duration.rb index e8a991026b5..573d4c25b91 100644 --- a/lib/gitlab/ci/pipeline/duration.rb +++ b/lib/gitlab/ci/pipeline/duration.rb @@ -82,6 +82,8 @@ module Gitlab module Duration extend self + STATUSES = %w[success failed running canceled].freeze + Period = Struct.new(:first, :last) do def duration last - first @@ -90,14 +92,15 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def from_pipeline(pipeline) - status = %w[success failed running canceled] - builds = pipeline.processables.latest - .where(status: status).where.not(started_at: nil).order(:started_at) + builds = + self_and_downstreams_builds_of_pipeline(pipeline) from_builds(builds) end # rubocop: enable CodeReuse/ActiveRecord + private + def from_builds(builds) now = Time.now @@ -113,8 +116,6 @@ module Gitlab process_duration(process_periods(periods)) end - private - def process_periods(periods) return periods if periods.empty? @@ -139,6 +140,20 @@ module Gitlab end # rubocop: disable CodeReuse/ActiveRecord + def self_and_downstreams_builds_of_pipeline(pipeline) + ::Ci::Build + .select(:id, :type, :started_at, :finished_at) + .in_pipelines( + pipeline.self_and_downstreams.select(:id) + ) + .with_status(STATUSES) + .latest + .where.not(started_at: nil) + .order(:started_at) + end + # rubocop: enable CodeReuse/ActiveRecord + + # rubocop: disable CodeReuse/ActiveRecord def process_duration(periods) periods.sum(&:duration) end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 484e18c6979..98f488d0f38 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -123,6 +123,7 @@ module Gitlab end @needs_attributes.flat_map do |need| + # We ignore the optional needed job in case it is excluded from the pipeline due to the job's rules. next if need[:optional] result = need_present?(need) diff --git a/lib/gitlab/ci/pipeline/seed/build/cache.rb b/lib/gitlab/ci/pipeline/seed/build/cache.rb index 409b6658cc0..936344b9ae8 100644 --- a/lib/gitlab/ci/pipeline/seed/build/cache.rb +++ b/lib/gitlab/ci/pipeline/seed/build/cache.rb @@ -16,6 +16,7 @@ module Gitlab @when = local_cache.delete(:when) @unprotect = local_cache.delete(:unprotect) @custom_key_prefix = custom_key_prefix + @fallback_keys = local_cache.delete(:fallback_keys) raise ArgumentError, "unknown cache keys: #{local_cache.keys}" if local_cache.any? end @@ -27,7 +28,8 @@ module Gitlab policy: @policy, untracked: @untracked, when: @when, - unprotect: @unprotect + unprotect: @unprotect, + fallback_keys: @fallback_keys }.compact end diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb index ded6877ef29..00b2ad58428 100644 --- a/lib/gitlab/ci/project_config.rb +++ b/lib/gitlab/ci/project_config.rb @@ -26,6 +26,7 @@ module Gitlab end delegate :content, :source, to: :@config, allow_nil: true + delegate :internal_include_prepended?, to: :@config def exists? !!@config&.exists? diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb index c6905f480a2..c5f010ebaea 100644 --- a/lib/gitlab/ci/project_config/auto_devops.rb +++ b/lib/gitlab/ci/project_config/auto_devops.rb @@ -13,6 +13,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :auto_devops_source end diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb index 0ed5d6fa226..0afdab23886 100644 --- a/lib/gitlab/ci/project_config/external_project.rb +++ b/lib/gitlab/ci/project_config/external_project.rb @@ -17,6 +17,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :external_project_source end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb index cf1292706d2..19cbf8e9c1e 100644 --- a/lib/gitlab/ci/project_config/remote.rb +++ b/lib/gitlab/ci/project_config/remote.rb @@ -12,6 +12,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :remote_source end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 435ad4d42fe..272425fd546 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -12,6 +12,10 @@ module Gitlab end end + def internal_include_prepended? + true + end + def source :repository_source end diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index ebe5728163b..9a4a6394fa1 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -24,6 +24,11 @@ module Gitlab raise NotImplementedError end + # Indicates if we are prepending the content with an "internal" `include` + def internal_include_prepended? + false + end + def source raise NotImplementedError end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 92a91854358..bf48c7d0bb7 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -29,12 +29,13 @@ module Gitlab attr_reader :signatures attr_reader :project_id attr_reader :original_data + attr_reader :found_by_pipeline delegate :file_path, :start_line, :end_line, to: :location alias_method :cve, :compare_key - def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false) # rubocop:disable Metrics/ParameterLists + def initialize(compare_key:, identifiers:, flags: [], links: [], remediations: [], location:, evidence:, metadata_version:, name:, original_data:, report_type:, scanner:, scan:, uuid:, confidence: nil, severity: nil, details: {}, signatures: [], project_id: nil, vulnerability_finding_signatures_enabled: false, found_by_pipeline: nil) # rubocop:disable Metrics/ParameterLists @compare_key = compare_key @confidence = confidence @identifiers = identifiers @@ -55,6 +56,7 @@ module Gitlab @signatures = signatures @project_id = project_id @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + @found_by_pipeline = found_by_pipeline @project_fingerprint = generate_project_fingerprint end @@ -188,6 +190,10 @@ module Gitlab original_data['assets'] || [] end + def raw_source_code_extract + original_data['raw_source_code_extract'] + end + # Returns either the max priority signature hex # or the location fingerprint def location_fingerprint diff --git a/lib/gitlab/ci/reports/security/report.rb b/lib/gitlab/ci/reports/security/report.rb index 54b21da5436..2287c397c2b 100644 --- a/lib/gitlab/ci/reports/security/report.rb +++ b/lib/gitlab/ci/reports/security/report.rb @@ -5,8 +5,9 @@ module Gitlab module Reports module Security class Report - attr_reader :created_at, :type, :pipeline, :findings, :scanners, :identifiers - attr_accessor :scan, :scanned_resources, :errors, :analyzer, :version, :schema_validation_status, :warnings + attr_reader :created_at, :type, :findings, :scanners, :identifiers + attr_accessor :scan, :pipeline, :scanned_resources, :errors, + :analyzer, :version, :schema_validation_status, :warnings delegate :project_id, to: :pipeline delegate :project, to: :pipeline diff --git a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb b/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb deleted file mode 100644 index 4be4cf62e7b..00000000000 --- a/lib/gitlab/ci/reports/security/vulnerability_reports_comparer.rb +++ /dev/null @@ -1,165 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Reports - module Security - class VulnerabilityReportsComparer - include Gitlab::Utils::StrongMemoize - - attr_reader :base_report, :head_report - - ACCEPTABLE_REPORT_AGE = 1.week - - def initialize(project, base_report, head_report) - @base_report = base_report - @head_report = head_report - - @signatures_enabled = project.licensed_feature_available?(:vulnerability_finding_signatures) - - if @signatures_enabled - @added_findings = [] - @fixed_findings = [] - calculate_changes - end - end - - def base_report_created_at - @base_report.created_at - end - - def head_report_created_at - @head_report.created_at - end - - def base_report_out_of_date - return false unless @base_report.created_at - - ACCEPTABLE_REPORT_AGE.ago > @base_report.created_at - end - - def added - strong_memoize(:added) do - if @signatures_enabled - @added_findings - else - head_report.findings - base_report.findings - end - end - end - - def fixed - strong_memoize(:fixed) do - if @signatures_enabled - @fixed_findings - else - base_report.findings - head_report.findings - end - end - end - - private - - def calculate_changes - # This is a deconstructed version of the eql? method on - # Ci::Reports::Security::Finding. It: - # - # * precomputes for the head_findings (using FindingMatcher): - # * sets of signature shas grouped by priority - # * mappings of signature shas to the head finding object - # - # These are then used when iterating the base findings to perform - # fast(er) prioritized, signature-based comparisons between each base finding - # and the head findings. - # - # Both the head_findings and base_findings arrays are iterated once - - base_findings = base_report.findings - head_findings = head_report.findings - - matcher = FindingMatcher.new(head_findings) - - base_findings.each do |base_finding| - next if base_finding.requires_manual_resolution? - - matched_head_finding = matcher.find_and_remove_match!(base_finding) - - @fixed_findings << base_finding if matched_head_finding.nil? - end - - @added_findings = matcher.unmatched_head_findings.values - end - end - - class FindingMatcher - attr_reader :unmatched_head_findings, :head_findings - - include Gitlab::Utils::StrongMemoize - - def initialize(head_findings) - @head_findings = head_findings - @unmatched_head_findings = @head_findings.index_by(&:object_id) - end - - def find_and_remove_match!(base_finding) - matched_head_finding = find_matched_head_finding_for(base_finding) - - # no signatures matched, so check the normal uuids of the base and head findings - # for a match - matched_head_finding = head_signatures_shas[base_finding.uuid] if matched_head_finding.nil? - - @unmatched_head_findings.delete(matched_head_finding.object_id) unless matched_head_finding.nil? - - matched_head_finding - end - - private - - def find_matched_head_finding_for(base_finding) - base_signature = sorted_signatures_for(base_finding).find do |signature| - # at this point a head_finding exists that has a signature with a - # matching priority, and a matching sha --> lookup the actual finding - # object from head_signatures_shas - head_signatures_shas[signature.signature_sha].eql?(base_finding) - end - - base_signature.present? ? head_signatures_shas[base_signature.signature_sha] : nil - end - - def sorted_signatures_for(base_finding) - base_finding.signatures.select { |signature| head_finding_signature?(signature) } - .sort_by { |sig| -sig.priority } - end - - def head_finding_signature?(signature) - head_signatures_priorities[signature.priority].include?(signature.signature_sha) - end - - def head_signatures_priorities - strong_memoize(:head_signatures_priorities) do - signatures_priorities = Hash.new { |hash, key| hash[key] = Set.new } - - head_findings.each_with_object(signatures_priorities) do |head_finding, memo| - head_finding.signatures.each do |signature| - memo[signature.priority].add(signature.signature_sha) - end - end - end - end - - def head_signatures_shas - strong_memoize(:head_signatures_shas) do - head_findings.each_with_object({}) do |head_finding, memo| - head_finding.signatures.each do |signature| - memo[signature.signature_sha] = head_finding - end - # for the final uuid check when no signatures have matched - memo[head_finding.uuid] = head_finding - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/resource_groups/logger.rb b/lib/gitlab/ci/resource_groups/logger.rb new file mode 100644 index 00000000000..9c93ee95bc7 --- /dev/null +++ b/lib/gitlab/ci/resource_groups/logger.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module ResourceGroups + class Logger < ::Gitlab::JsonLogger + def self.file_name_noext + 'ci_resource_groups_json' + end + end + end + end +end diff --git a/lib/gitlab/ci/runner_releases.rb b/lib/gitlab/ci/runner_releases.rb index dab24bfd501..a27dd3896e1 100644 --- a/lib/gitlab/ci/runner_releases.rb +++ b/lib/gitlab/ci/runner_releases.rb @@ -15,9 +15,14 @@ module Gitlab reset_backoff! end + def enabled? + ::Gitlab::CurrentSettings.current_application_settings.update_runner_versions_enabled? + end + # Returns a sorted list of the publicly available GitLab Runner releases # def releases + return unless enabled? return if backoff_active? Rails.cache.fetch( diff --git a/lib/gitlab/ci/secure_files/cer.rb b/lib/gitlab/ci/secure_files/cer.rb index 45d2898c29b..3340afa8f2a 100644 --- a/lib/gitlab/ci/secure_files/cer.rb +++ b/lib/gitlab/ci/secure_files/cer.rb @@ -36,7 +36,7 @@ module Gitlab private def expires_at - certificate_data.not_before + certificate_data.not_after end def id diff --git a/lib/gitlab/ci/secure_files/p12.rb b/lib/gitlab/ci/secure_files/p12.rb index 1006a4d05b2..04cd4243bb0 100644 --- a/lib/gitlab/ci/secure_files/p12.rb +++ b/lib/gitlab/ci/secure_files/p12.rb @@ -36,7 +36,7 @@ module Gitlab private def expires_at - certificate_data.not_before + certificate_data.not_after end def serial diff --git a/lib/gitlab/ci/status/build/erased.rb b/lib/gitlab/ci/status/build/erased.rb index d74cfc1ee77..c3430b8cc1c 100644 --- a/lib/gitlab/ci/status/build/erased.rb +++ b/lib/gitlab/ci/status/build/erased.rb @@ -7,8 +7,8 @@ module Gitlab class Erased < Status::Extended def illustration { - image: 'illustrations/erased-log_empty.svg', - size: 'svg-430', + image: 'illustrations/empty-state/empty-projects-deleted-md.svg', + size: 'svg-150', title: _('Job has been erased') } end diff --git a/lib/gitlab/ci/status/build/factory.rb b/lib/gitlab/ci/status/build/factory.rb index a4434e2c144..54f6784b847 100644 --- a/lib/gitlab/ci/status/build/factory.rb +++ b/lib/gitlab/ci/status/build/factory.rb @@ -11,12 +11,12 @@ module Gitlab Status::Build::Manual, Status::Build::Canceled, Status::Build::Created, - Status::Build::WaitingForResource, Status::Build::Preparing, Status::Build::Pending, Status::Build::Skipped, Status::Build::WaitingForApproval], - [Status::Build::Cancelable, + [Status::Build::WaitingForResource, + Status::Build::Cancelable, Status::Build::Retryable], [Status::Build::FailedUnmetPrerequisites, Status::Build::Failed], diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index e854164d377..1ba78b357e5 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -7,17 +7,19 @@ module Gitlab include Gitlab::Utils::StrongMemoize # This class accepts an array of arrays/hashes/or objects - def initialize(all_statuses, with_allow_failure: true, dag: false) - unless all_statuses.respond_to?(:pluck) - raise ArgumentError, "all_statuses needs to respond to `.pluck`" + # `with_allow_failure` will be removed when deleting ci_remove_ensure_stage_service + def initialize(all_jobs, with_allow_failure: true, dag: false, project: nil) + unless all_jobs.respond_to?(:pluck) + raise ArgumentError, "all_jobs needs to respond to `.pluck`" end @status_set = Set.new @status_key = 0 @allow_failure_key = 1 if with_allow_failure @dag = dag + @project = project - consume_all_statuses(all_statuses) + consume_all_jobs(all_jobs) end # The status calculation is order dependent, @@ -26,6 +28,14 @@ module Gitlab # 2. In other cases we assume that status is of that type # based on what statuses are no longer valid based on the # data set that we have + # + # This method is used for three cases: + # 1. When it is called for a stage or a pipeline (with `all_jobs` from all jobs in a stage or a pipeline), + # then, the returned status is assigned to the stage or pipeline. + # 2. When it is called for a job (with `all_jobs` from all previous jobs or all needed jobs), + # then, the returned status is used to determine if the job is processed or not. + # 3. When it is called for a group (of jobs that are related), + # then, the returned status is used to show the overall status of the group. # rubocop: disable Metrics/CyclomaticComplexity # rubocop: disable Metrics/PerceivedComplexity def status @@ -35,9 +45,6 @@ module Gitlab if @dag && any_skipped_or_ignored? # The DAG job is skipped if one of the needs does not run at all. 'skipped' - elsif @dag && !only_of?(:success, :failed, :canceled, :skipped, :success_with_warnings) - # DAG is blocked from executing if a dependent is not "complete" - 'pending' elsif only_of?(:skipped, :ignored) 'skipped' elsif only_of?(:success, :skipped, :success_with_warnings, :ignored) @@ -94,42 +101,41 @@ module Gitlab any_of?(:skipped) || any_of?(:ignored) end - def consume_all_statuses(all_statuses) + def consume_all_jobs(all_jobs) columns = [] columns[@status_key] = :status columns[@allow_failure_key] = :allow_failure if @allow_failure_key - all_statuses + all_jobs .pluck(*columns) # rubocop: disable CodeReuse/ActiveRecord - .each(&method(:consume_status)) + .each do |job_attrs| + consume_job_status(Array.wrap(job_attrs)) + end end - def consume_status(description) - # convert `"status"` into `["status"]` - description = Array(description) - - status = - if success_with_warnings?(description) + def consume_job_status(job_attrs) + status_result = + if success_with_warnings?(job_attrs) :success_with_warnings - elsif ignored_status?(description) + elsif ignored_status?(job_attrs) :ignored else - description[@status_key].to_sym + job_attrs[@status_key].to_sym end - @status_set.add(status) + @status_set.add(status_result) end - def success_with_warnings?(status) + def success_with_warnings?(job_attrs) @allow_failure_key && - status[@allow_failure_key] && - ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(status[@status_key]) + job_attrs[@allow_failure_key] && + ::Ci::HasStatus::PASSED_WITH_WARNINGS_STATUSES.include?(job_attrs[@status_key]) end - def ignored_status?(status) + def ignored_status?(job_attrs) @allow_failure_key && - status[@allow_failure_key] && - ::Ci::HasStatus::EXCLUDE_IGNORED_STATUSES.include?(status[@status_key]) + job_attrs[@allow_failure_key] && + ::Ci::HasStatus::IGNORED_STATUSES.include?(job_attrs[@status_key]) end end end diff --git a/lib/gitlab/ci/status/processable/waiting_for_resource.rb b/lib/gitlab/ci/status/processable/waiting_for_resource.rb index c9b1dd795d0..ac82c99b5f1 100644 --- a/lib/gitlab/ci/status/processable/waiting_for_resource.rb +++ b/lib/gitlab/ci/status/processable/waiting_for_resource.rb @@ -17,9 +17,39 @@ module Gitlab } end + def has_action? + current_processable.present? + end + + def action_icon + nil + end + + def action_title + nil + end + + def action_button_title + _('View job currently using resource') + end + + def action_path + project_job_path(subject.project, current_processable) + end + + def action_method + :get + end + def self.matches?(processable, _) processable.waiting_for_resource? end + + private + + def current_processable + @current_processable ||= subject.resource_group.current_processable + end end end end diff --git a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml index 11420b05dfb..4f12f0cd3b8 100644 --- a/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Auto-DevOps.gitlab-ci.yml @@ -58,7 +58,6 @@ variables: POSTGRES_USER: user POSTGRES_PASSWORD: testing-password - POSTGRES_ENABLED: "true" POSTGRES_DB: $CI_ENVIRONMENT_SLUG DOCKER_DRIVER: overlay2 diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 40f5109851b..7a4c65f8c5b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 40f5109851b..7a4c65f8c5b 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.28.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.32.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml index 47b79302828..b2ab6704e35 100644 --- a/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Code-Quality.gitlab-ci.yml @@ -8,7 +8,7 @@ code_quality: variables: DOCKER_DRIVER: overlay2 DOCKER_TLS_CERTDIR: "" - CODE_QUALITY_IMAGE_TAG: "0.89.0" + CODE_QUALITY_IMAGE_TAG: "0.94.0" CODE_QUALITY_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/ci-cd/codequality:$CODE_QUALITY_IMAGE_TAG" needs: [] script: diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml index fa609afc5a8..192d06bfa14 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.gitlab-ci.yml @@ -22,7 +22,8 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6" + CS_SCHEMA_MODEL: 15 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" @@ -39,12 +40,12 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: - gtcs scan rules: - - if: $CONTAINER_SCANNING_DISABLED + - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH && $CI_GITLAB_FIPS_MODE == "true" && diff --git a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml index f750bda2a3f..9a4c75e7402 100644 --- a/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Container-Scanning.latest.gitlab-ci.yml @@ -22,7 +22,8 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables variables: - CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:6" + CS_SCHEMA_MODEL: 15 container_scanning: image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" @@ -39,12 +40,12 @@ container_scanning: reports: container_scanning: gl-container-scanning-report.json dependency_scanning: gl-dependency-scanning-report.json - paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json, "**/gl-sbom-*.cdx.json"] dependencies: [] script: - gtcs scan rules: - - if: $CONTAINER_SCANNING_DISABLED + - if: $CONTAINER_SCANNING_DISABLED == 'true' || $CONTAINER_SCANNING_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index aa2356f6a34..4ee5fa74df9 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml index eb8e5de5b56..63cf265fc6e 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" - DS_MAJOR_VERSION: 3 + DS_MAJOR_VERSION: 4 DS_SCHEMA_MODEL: 15 dependency_scanning: @@ -56,15 +56,16 @@ dependency_scanning: .gemnasium-shared-rule: exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + - '**/Gemfile.lock' + - '**/composer.lock' + - '**/gems.locked' + - '**/go.sum' + - '**/npm-shrinkwrap.json' + - '**/package-lock.json' + - '**/yarn.lock' + - '**/pnpm-lock.yaml' + - '**/packages.lock.json' + - '**/conan.lock' gemnasium-dependency_scanning: extends: @@ -74,7 +75,7 @@ gemnasium-dependency_scanning: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ when: never @@ -91,10 +92,10 @@ gemnasium-dependency_scanning: .gemnasium-maven-shared-rule: exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + - '**/build.gradle' + - '**/build.gradle.kts' + - '**/build.sbt' + - '**/pom.xml' gemnasium-maven-dependency_scanning: extends: @@ -103,7 +104,7 @@ gemnasium-maven-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ when: never @@ -119,12 +120,13 @@ gemnasium-maven-dependency_scanning: .gemnasium-python-shared-rule: exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + - '**/requirements.txt' + - '**/requirements.pip' + - '**/Pipfile' + - '**/Pipfile.lock' + - '**/requires.txt' + - '**/setup.py' + - '**/poetry.lock' gemnasium-python-dependency_scanning: extends: @@ -133,7 +135,7 @@ gemnasium-python-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-python" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml index 655ac6ee712..4d7c3930741 100644 --- a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" DS_EXCLUDED_ANALYZERS: "" DS_EXCLUDED_PATHS: "spec, test, tests, tmp" - DS_MAJOR_VERSION: 3 + DS_MAJOR_VERSION: 4 DS_SCHEMA_MODEL: 15 dependency_scanning: @@ -56,15 +56,16 @@ dependency_scanning: .gemnasium-shared-rule: exists: - - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' - - '{composer.lock,*/composer.lock,*/*/composer.lock}' - - '{gems.locked,*/gems.locked,*/*/gems.locked}' - - '{go.sum,*/go.sum,*/*/go.sum}' - - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' - - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' - - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' - - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' - - '{conan.lock,*/conan.lock,*/*/conan.lock}' + - '**/Gemfile.lock' + - '**/composer.lock' + - '**/gems.locked' + - '**/go.sum' + - '**/npm-shrinkwrap.json' + - '**/package-lock.json' + - '**/yarn.lock' + - '**/pnpm-lock.yaml' + - '**/packages.lock.json' + - '**/conan.lock' gemnasium-dependency_scanning: extends: @@ -74,7 +75,7 @@ gemnasium-dependency_scanning: DS_ANALYZER_NAME: "gemnasium" GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ when: never @@ -109,10 +110,10 @@ gemnasium-dependency_scanning: .gemnasium-maven-shared-rule: exists: - - '{build.gradle,*/build.gradle,*/*/build.gradle}' - - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' - - '{build.sbt,*/build.sbt,*/*/build.sbt}' - - '{pom.xml,*/pom.xml,*/*/pom.xml}' + - '**/build.gradle' + - '**/build.gradle.kts' + - '**/build.sbt' + - '**/pom.xml' gemnasium-maven-dependency_scanning: extends: @@ -121,7 +122,7 @@ gemnasium-maven-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-maven" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ when: never @@ -155,12 +156,13 @@ gemnasium-maven-dependency_scanning: .gemnasium-python-shared-rule: exists: - - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' - - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' - - '{Pipfile,*/Pipfile,*/*/Pipfile}' - - '{requires.txt,*/requires.txt,*/*/requires.txt}' - - '{setup.py,*/setup.py,*/*/setup.py}' - - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + - '**/requirements.txt' + - '**/requirements.pip' + - '**/Pipfile' + - '**/Pipfile.lock' + - '**/requires.txt' + - '**/setup.py' + - '**/poetry.lock' gemnasium-python-dependency_scanning: extends: @@ -169,7 +171,7 @@ gemnasium-python-dependency_scanning: variables: DS_ANALYZER_NAME: "gemnasium-python" rules: - - if: $DEPENDENCY_SCANNING_DISABLED + - if: $DEPENDENCY_SCANNING_DISABLED == 'true' || $DEPENDENCY_SCANNING_DISABLED == '1' when: never - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 372b782c0a0..622b44d78ad 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index feba2efcf22..2954ddf8a35 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.46.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.48.2' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml index f8668699fe5..b1c81e9ed5b 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml @@ -32,7 +32,7 @@ license_scanning: license_scanning: gl-license-scanning-report.json dependencies: [] rules: - - if: $LICENSE_MANAGEMENT_DISABLED + - if: $LICENSE_MANAGEMENT_DISABLED == 'true' || $LICENSE_MANAGEMENT_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml index e47f669c2e2..8e1b0159cb0 100644 --- a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml @@ -32,7 +32,7 @@ license_scanning: license_scanning: gl-license-scanning-report.json dependencies: [] rules: - - if: $LICENSE_MANAGEMENT_DISABLED + - if: $LICENSE_MANAGEMENT_DISABLED == 'true' || $LICENSE_MANAGEMENT_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index 12105e0e95d..a849d36a5b8 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -6,7 +6,7 @@ load_performance: DOCKER_TLS_CERTDIR: "" K6_IMAGE: grafana/k6 K6_VERSION: 0.41.0 - K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/examples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml index c195ecd8ee5..a64e1e4a40f 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.gitlab-ci.yml @@ -31,10 +31,10 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 77048037915..77f2c5a8c99 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -31,10 +31,10 @@ kics-iac-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kics:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index 8b49d2de8cf..d567ab2a141 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -1,7 +1,7 @@ # Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/sast/ # # Configure SAST with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). -# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-variables +# List of available variables: https://docs.gitlab.com/ee/user/application_security/sast/index.html#available-cicd-variables variables: # Setting this variable will affect all Security templates @@ -48,10 +48,10 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ when: never @@ -74,10 +74,10 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ when: never @@ -95,10 +95,10 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ when: never @@ -119,13 +119,13 @@ gosec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" mobsf-android-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -138,7 +138,7 @@ mobsf-android-sast: mobsf-ios-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -153,10 +153,10 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never @@ -169,10 +169,10 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never @@ -185,10 +185,10 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never @@ -198,20 +198,12 @@ pmd-apex-sast: security-code-scan-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: '3' - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 15.9 and removed in GitLab 16.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/390416" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.csproj' - - '**/*.vbproj' + - when: never semgrep-sast: extends: .sast-analyzer @@ -219,10 +211,10 @@ semgrep-sast: name: "$SAST_ANALYZER_IMAGE" variables: SEARCH_MAX_DEPTH: 20 - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never @@ -246,10 +238,10 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never @@ -262,7 +254,7 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ @@ -271,7 +263,7 @@ spotbugs-sast: exists: - '**/AndroidManifest.xml' when: never - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH exists: diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index 1c4dbe6cd0f..88d10f8b235 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -48,10 +48,10 @@ brakeman-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/brakeman:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /brakeman/ when: never @@ -80,10 +80,10 @@ flawfinder-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/flawfinder:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /flawfinder/ when: never @@ -120,10 +120,10 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/kubesec:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kubesec/ when: never @@ -141,13 +141,13 @@ kubesec-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/mobsf:$SAST_ANALYZER_IMAGE_TAG" mobsf-android-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -169,7 +169,7 @@ mobsf-android-sast: mobsf-ios-sast: extends: .mobsf-sast rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /mobsf/ when: never @@ -193,10 +193,10 @@ nodejs-scan-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/nodejs-scan:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never @@ -214,10 +214,10 @@ phpcs-security-audit-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/phpcs-security-audit:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never @@ -235,10 +235,10 @@ pmd-apex-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/pmd-apex:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never @@ -253,26 +253,12 @@ pmd-apex-sast: security-code-scan-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 3 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/security-code-scan:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 15.9 and removed in GitLab 16.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/390416" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.csproj' - - '**/*.vbproj' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.csproj' - - '**/*.vbproj' + - when: never semgrep-sast: extends: .sast-analyzer @@ -280,10 +266,10 @@ semgrep-sast: name: "$SAST_ANALYZER_IMAGE" variables: SEARCH_MAX_DEPTH: 20 - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never @@ -323,10 +309,10 @@ sobelow-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/sobelow:$SAST_ANALYZER_IMAGE_TAG" rules: - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never @@ -344,7 +330,7 @@ spotbugs-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SAST_ANALYZER_IMAGE_TAG: 3 + SAST_ANALYZER_IMAGE_TAG: 4 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/spotbugs:$SAST_ANALYZER_IMAGE_TAG" rules: - if: $SAST_EXCLUDED_ANALYZERS =~ /spotbugs/ @@ -353,7 +339,7 @@ spotbugs-sast: exists: - '**/AndroidManifest.xml' when: never - - if: $SAST_DISABLED + - if: $SAST_DISABLED == 'true' || $SAST_DISABLED == '1' when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml index b7a9dbf7bc6..9d0b904117a 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" - SECRETS_ANALYZER_VERSION: "4" + SECRETS_ANALYZER_VERSION: "5" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: @@ -27,7 +27,7 @@ variables: secret_detection: extends: .secret-analyzer rules: - - if: $SECRET_DETECTION_DISABLED + - if: $SECRET_DETECTION_DISABLED == 'true' || $SECRET_DETECTION_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH script: diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml index 6603ee4268e..56a8ad794dc 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -8,7 +8,7 @@ variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" SECRET_DETECTION_IMAGE_SUFFIX: "" - SECRETS_ANALYZER_VERSION: "4" + SECRETS_ANALYZER_VERSION: "5" SECRET_DETECTION_EXCLUDED_PATHS: "" .secret-analyzer: @@ -27,7 +27,7 @@ variables: secret_detection: extends: .secret-analyzer rules: - - if: $SECRET_DETECTION_DISABLED + - if: $SECRET_DETECTION_DISABLED == 'true' || $SECRET_DETECTION_DISABLED == '1' when: never - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. diff --git a/lib/gitlab/ci/templates/Python.gitlab-ci.yml b/lib/gitlab/ci/templates/Python.gitlab-ci.yml index febbb36d834..d53f3ddcad4 100644 --- a/lib/gitlab/ci/templates/Python.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Python.gitlab-ci.yml @@ -23,26 +23,24 @@ cache: - venv/ before_script: - - python --version # For debugging + - python --version ; pip --version # For debugging - pip install virtualenv - virtualenv venv - source venv/bin/activate test: script: - - python setup.py test - - pip install tox flake8 # you can also use tox - - tox -e py36,flake8 + - pip install ruff tox # you can also use tox + - pip install --editable ".[test]" + - tox -e py,ruff run: script: - - python setup.py bdist_wheel - # an alternative approach is to install and run: - - pip install dist/* + - pip install . # run the command here artifacts: paths: - - dist/*.whl + - build/* pages: script: diff --git a/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml new file mode 100644 index 00000000000..d9bc76dad1e --- /dev/null +++ b/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml @@ -0,0 +1,66 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/API-Discovery.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/api_discovery/ +# +# Configure API Discovery with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/api_discovery/#available-cicd-variables + +variables: + API_DISCOVERY_PACKAGES: "$CI_API_V4_URL/projects/42503323/packages" + API_DISCOVERY_VERSION: "1" + +.api_discovery_java_spring_boot: + stage: test + allow_failure: true + script: + # + # Check configuration + - if [[ -z "$API_DISCOVERY_VERSION" ]]; then echo "Error, API_DISCOVERY_VERSION not provided. Please set this variable and re-run the pipeline."; exit 1; fi + # + # Check for required commands + - requires() { command -v "$1" >/dev/null 2>&1 || { echo "'$1' is required but it's not installed. Add the needed command to the job image and retry." >&2; exit 1; } } + - requires 'curl' + - requires 'java' + # + # Set JAVA_HOME if API_DISCOVERY_JAVA_HOME provided + - if [[ -n "$API_DISCOVERY_JAVA_HOME" ]]; then export JAVA_HOME="$API_DISCOVERY_JAVA_HOME"; export PATH="$JAVA_HOME/bin:$PATH"; fi + # + # Download jar file + - if [[ -n "$API_DISCOVERY_PACKAGE_TOKEN" ]]; then echo "Using API_DISCOVERY_PACKAGE_TOKEN"; export CURL_AUTH="-H PRIVATE-TOKEN:$API_DISCOVERY_PACKAGE_TOKEN"; else export CURL_AUTH=""; fi + - DL_URL="$API_DISCOVERY_PACKAGES/maven/com/gitlab/analyzers/api-discovery/api-discovery_spring-boot/$API_DISCOVERY_VERSION/api-discovery_spring-boot-$API_DISCOVERY_VERSION.jar" + - echo "Downloading Discovery jar from '${DL_URL}'" + - CURL_CMD="curl -L ${CURL_AUTH} --write-out "%{http_code}" --output api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar ${DL_URL}" + - STATUS_CODE=$(${CURL_CMD}) + - RC=$? + - if [[ $RC -ne 0 ]]; then echo "Error connecting to GitLab API, curl exit code was $RC."; echo "To diagnose, see the curl documentation- https://everything.curl.dev/usingcurl/returns"; exit 1; fi + - if [[ "$STATUS_CODE" != "200" ]]; then echo "Error, Unable to download api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar"; echo "Error, Status Code was $STATUS_CODE, but wanted 200"; exit 1; fi + # + # Run API Discovery + - java -jar "api_discovery_java_spring_boot_${API_DISCOVERY_VERSION}.jar" + # + # Check for expected output file + - if [[ ! -e "gl-api-discovery-openapi.json" ]]; then echo "Error, Unable to find gl-api-discovery-openapi.json"; exit 1; fi + # + artifacts: + when: always + paths: + - gl-api-discovery-openapi.json + - gl-*.log + rules: + - if: $API_DISCOVERY_DISABLED + when: never + - if: $API_DISCOVERY_DISABLED_FOR_DEFAULT_BRANCH && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml index cdfa4556769..544aee904d5 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # - FUZZAPI_VERSION: "2" + FUZZAPI_VERSION: "3" FUZZAPI_IMAGE_SUFFIX: "" FUZZAPI_IMAGE: api-security @@ -35,9 +35,12 @@ apifuzzer_fuzz: image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && diff --git a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml index f12efa1db34..feaa2965339 100644 --- a/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/API-Fuzzing.latest.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # - FUZZAPI_VERSION: "2" + FUZZAPI_VERSION: "3" FUZZAPI_IMAGE_SUFFIX: "" FUZZAPI_IMAGE: api-security @@ -35,9 +35,12 @@ apifuzzer_fuzz: image: $SECURE_ANALYZERS_PREFIX/$FUZZAPI_IMAGE:$FUZZAPI_VERSION$FUZZAPI_IMAGE_SUFFIX allow_failure: true rules: - - if: $API_FUZZING_DISABLED + - if: $API_FUZZING_DISABLED == 'true' || $API_FUZZING_DISABLED == '1' when: never - - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH && + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $API_FUZZING_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never diff --git a/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml new file mode 100644 index 00000000000..b626a7ca770 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml @@ -0,0 +1,65 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/BAS.latest.gitlab-ci.yml + +# To use this template, add the following to your .gitlab-ci.yml file: +# +# include: +# template: BAS.latest.gitlab-ci.yml +# +# You also need to add a `dast` stage to your `stages:` configuration. A sample configuration for DAST: +# +# stages: +# - build +# - test +# - deploy +# - dast +# +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/breach_and_attack_simulation/index.html#extend-dynamic-application-security-testing-dast + +# Include the DAST.latest template if $DAST_VERSION is null because this means a DAST template has not been included already. +include: + - template: Security/DAST.latest.gitlab-ci.yml + rules: + - if: $DAST_VERSION == null + +variables: + BAS_CALLBACK_IMAGE_TAG: "latest" + BAS_DAST_IMAGE_TAG: "latest" + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + +dast_with_bas: + extends: + - dast + - .dast_with_bas + rules: + # Don't add if the DAST+BAS job is disabled. + - if: $DAST_BAS_DISABLED == 'true' || $DAST_BAS_DISABLED == '1' + when: never + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + # If there's no open merge request, add it to a *branch* pipeline instead. + - if: $CI_COMMIT_BRANCH + +.dast_with_bas: + image: + name: "$SECURE_ANALYZERS_PREFIX/dast/breach-and-attack-simulation:$BAS_DAST_IMAGE_TAG" + variables: + DAST_BROWSER_SCAN: "true" + DAST_FF_ENABLE_BAS: "true" + DAST_FULL_SCAN_ENABLED: "true" + +.dast_with_bas_using_services: + extends: .dast_with_bas + services: + - name: "$SECURE_ANALYZERS_PREFIX/callback:$BAS_CALLBACK_IMAGE_TAG" + alias: callback + variables: + DAST_BROWSER_CALLBACK: "Address:http://callback" + FF_NETWORK_PER_BUILD: "true" diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml index 89944e347f6..1f11ec8e288 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.gitlab-ci.yml @@ -49,6 +49,6 @@ coverage_fuzzing_unlicensed: coverage_fuzzing: gl-coverage-fuzzing-report.json when: always rules: - - if: $COVFUZZ_DISABLED + - if: $COVFUZZ_DISABLED == 'true' || $COVFUZZ_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bcoverage_fuzzing\b/ diff --git a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml index 4f6ba427058..0cf52468067 100644 --- a/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Coverage-Fuzzing.latest.gitlab-ci.yml @@ -49,7 +49,7 @@ coverage_fuzzing_unlicensed: coverage_fuzzing: gl-coverage-fuzzing-report.json when: always rules: - - if: $COVFUZZ_DISABLED + - if: $COVFUZZ_DISABLED == 'true' || $COVFUZZ_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml index 1b33596baa0..ee99d3b4614 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # - DAST_API_VERSION: "2" + DAST_API_VERSION: "3" DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-security @@ -35,9 +35,12 @@ dast_api: image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never - if: $CI_COMMIT_BRANCH && diff --git a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml index a28914d082f..f0b3dc3d2d9 100644 --- a/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-API.latest.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" # - DAST_API_VERSION: "2" + DAST_API_VERSION: "3" DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-security @@ -35,9 +35,12 @@ dast_api: image: $SECURE_ANALYZERS_PREFIX/$DAST_API_IMAGE:$DAST_API_VERSION$DAST_API_IMAGE_SUFFIX allow_failure: true rules: - - if: $DAST_API_DISABLED + - if: $DAST_API_DISABLED == 'true' || $DAST_API_DISABLED == '1' when: never - - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == 'true' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $DAST_API_DISABLED_FOR_DEFAULT_BRANCH == '1' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 5863da142f0..7b9d16e4192 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -14,7 +14,7 @@ stages: variables: SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" - DAST_API_VERSION: "2" + DAST_API_VERSION: "3" DAST_API_IMAGE_SUFFIX: "" DAST_API_IMAGE: api-security diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 733ba4e4954..1ed4cd86e82 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -13,7 +13,7 @@ stages: - dast variables: - DAST_VERSION: 3 + DAST_VERSION: 4 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" diff --git a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml index c43296b5865..792bd7f666b 100644 --- a/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 3 + DAST_VERSION: 4 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" @@ -42,13 +42,23 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_DISABLED + - if: $DAST_DISABLED == 'true' || $DAST_DISABLED == '1' when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED == 'true' + when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED + $REVIEW_DISABLED == '1' when: never - if: $CI_COMMIT_BRANCH && $GITLAB_FEATURES =~ /\bdast\b/ + after_script: + # Remove any debug.log files because they might contain secrets. + - rm -f /zap/wrk/**/debug.log + - cp -r /zap/wrk dast_artifacts diff --git a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml index 27bcc14bcf5..d1d1c4d7e52 100644 --- a/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ # List of available variables: https://docs.gitlab.com/ee/user/application_security/dast/#available-variables variables: - DAST_VERSION: 3 + DAST_VERSION: 4 # Setting this variable will affect all Security templates # (SAST, Dependency Scanning, ...) SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" @@ -44,13 +44,19 @@ dast: reports: dast: gl-dast-report.json rules: - - if: $DAST_DISABLED + - if: $DAST_DISABLED == 'true' || $DAST_DISABLED == '1' when: never - - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH && + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == 'true' && $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME when: never + - if: $DAST_DISABLED_FOR_DEFAULT_BRANCH == '1' && + $CI_DEFAULT_BRANCH == $CI_COMMIT_REF_NAME + when: never + - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && + $REVIEW_DISABLED == 'true' + when: never - if: $CI_DEFAULT_BRANCH != $CI_COMMIT_REF_NAME && - $REVIEW_DISABLED + $REVIEW_DISABLED == '1' when: never # Add the job to merge request pipelines if there's an open merge request. diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml index 631f6cecddf..9a43713cc26 100644 --- a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -255,7 +255,7 @@ dast-runner-validation: api-security: extends: .download_images variables: - SECURE_BINARIES_ANALYZER_VERSION: "2" + SECURE_BINARIES_ANALYZER_VERSION: "3" only: variables: - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 51bcbd278d5..2661c208665 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -24,6 +24,9 @@ validate: build: extends: .terraform:build + environment: + name: $TF_STATE_NAME + action: prepare deploy: extends: .terraform:deploy @@ -31,3 +34,4 @@ deploy: - build environment: name: $TF_STATE_NAME + action: start diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index dd1676f25b6..f16c28e7b60 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -9,7 +9,7 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.1:v0.43.0" + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.4:v1.0.0" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project @@ -23,24 +23,24 @@ cache: .terraform:fmt: &terraform_fmt stage: validate script: - - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true .terraform:validate: &terraform_validate stage: validate script: - - cd "${TF_ROOT}" - gitlab-terraform validate .terraform:build: &terraform_build stage: build script: - - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} artifacts: + # The next line, which disables public access to pipeline artifacts, may not be available everywhere. + # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic + public: false paths: - ${TF_ROOT}/plan.cache reports: @@ -49,17 +49,16 @@ cache: .terraform:deploy: &terraform_deploy stage: deploy script: - - cd "${TF_ROOT}" - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $TF_AUTO_DEPLOY == "true" - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual .terraform:destroy: &terraform_destroy stage: cleanup script: - - cd "${TF_ROOT}" - gitlab-terraform destroy resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index bc23a7c2a95..88fe55a44ab 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -24,7 +24,6 @@ cache: .terraform:fmt: &terraform_fmt stage: validate script: - - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true rules: @@ -36,7 +35,6 @@ cache: .terraform:validate: &terraform_validate stage: validate script: - - cd "${TF_ROOT}" - gitlab-terraform validate rules: - if: $CI_PIPELINE_SOURCE == "merge_request_event" @@ -47,11 +45,13 @@ cache: .terraform:build: &terraform_build stage: build script: - - cd "${TF_ROOT}" - gitlab-terraform plan - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} artifacts: + # The next line, which disables public access to pipeline artifacts, may not be available everywhere. + # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic + public: false paths: - ${TF_ROOT}/plan.cache reports: @@ -65,7 +65,6 @@ cache: .terraform:deploy: &terraform_deploy stage: deploy script: - - cd "${TF_ROOT}" - gitlab-terraform apply resource_group: ${TF_STATE_NAME} rules: @@ -76,7 +75,6 @@ cache: .terraform:destroy: &terraform_destroy stage: cleanup script: - - cd "${TF_ROOT}" - gitlab-terraform destroy resource_group: ${TF_STATE_NAME} when: manual diff --git a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml index e73e6194760..6d5bd7c2172 100644 --- a/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Module-Base.gitlab-ci.yml @@ -14,7 +14,7 @@ variables: TERRAFORM_MODULE_DIR: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project. TERRAFORM_MODULE_NAME: ${CI_PROJECT_NAME} # The name of your Terraform module, must not have any spaces or underscores (will be translated to hyphens). TERRAFORM_MODULE_SYSTEM: local # The system or provider your Terraform module targets (ex. local, aws, google). - TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning. + TERRAFORM_MODULE_VERSION: ${CI_COMMIT_TAG} # The version - it's recommended to follow SemVer for Terraform Module Versioning. .terraform-module:fmt: stage: validate @@ -29,7 +29,7 @@ variables: stage: deploy image: $CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/stable:latest script: - - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens + - TERRAFORM_MODULE_NAME=$(echo "${TERRAFORM_MODULE_NAME}" | tr " _" -) # module-name must not have spaces or underscores, so translate them to hyphens # Builds the Terraform module artifact: a gzipped tar archive with the contents from `$TERRAFORM_MODULE_DIR` without a `.git` directory. - tar -vczf /tmp/${TERRAFORM_MODULE_NAME}-${TERRAFORM_MODULE_SYSTEM}-${TERRAFORM_MODULE_VERSION}.tgz -C ${TERRAFORM_MODULE_DIR} --exclude=./.git . # Uploads the Terraform module artifact to the GitLab Terraform Module Registry, see diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index a907915587a..e8d1eb8e1c4 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -17,7 +17,7 @@ load_performance: variables: K6_IMAGE: grafana/k6 K6_VERSION: 0.41.0 - K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/examples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 8dfb6c38b55..59b45d865f1 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -80,7 +80,7 @@ deploy_job: # the artifact files will be copied to: # P:\Projects\YourApp\Builds\Rev1.0.0.1 - First commit\ - '$commitSubject = git log -1 --pretty=%s' - - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_BUILD_TAG) + " - " + $commitSubject + "\"' + - '$deployFolder = $($env:DEPLOY_FOLDER) + "\" + $($env:CI_COMMIT_TAG) + " - " + $commitSubject + "\"' # xcopy takes care of recursively creating required folders - 'xcopy /y ".\$env:EXE_RELEASE_FOLDER\YourApp.exe" "$deployFolder"' diff --git a/lib/gitlab/ci/trace/chunked_io.rb b/lib/gitlab/ci/trace/chunked_io.rb index 32f64948635..a3f1b472710 100644 --- a/lib/gitlab/ci/trace/chunked_io.rb +++ b/lib/gitlab/ci/trace/chunked_io.rb @@ -166,13 +166,6 @@ module Gitlab end def destroy! - # TODO: Remove this logging once we confirmed new live trace architecture is functional. - # See https://gitlab.com/gitlab-com/gl-infra/infrastructure/issues/4667. - unless build.has_archived_trace? - Sidekiq.logger.warn(message: 'The job does not have archived trace but going to be destroyed.', - job_id: build.id) - end - trace_chunks.fast_destroy_all @tell = @size = 0 ensure diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 89d681c418d..86e54fdfcdf 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -140,11 +140,13 @@ module Gitlab # Set environment name here so we can access it when evaluating the job's rules variables.append(key: 'CI_ENVIRONMENT_NAME', value: job.environment) if job.environment - # legacy variables - variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) - variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request - variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? + if Feature.disabled?(:ci_remove_legacy_predefined_variables, project) + # legacy variables + variables.append(key: 'CI_BUILD_NAME', value: job.name) + variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) + variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request + variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? + end end end diff --git a/lib/gitlab/ci/variables/builder/pipeline.rb b/lib/gitlab/ci/variables/builder/pipeline.rb index 96d6f1673b9..1e7a18d70b0 100644 --- a/lib/gitlab/ci/variables/builder/pipeline.rb +++ b/lib/gitlab/ci/variables/builder/pipeline.rb @@ -40,7 +40,7 @@ module Gitlab attr_reader :pipeline - def predefined_commit_variables + def predefined_commit_variables # rubocop:disable Metrics/AbcSize - Remove this rubocop:disable when FF `ci_remove_legacy_predefined_variables` is removed. Gitlab::Ci::Variables::Collection.new.tap do |variables| next variables unless pipeline.sha.present? @@ -57,7 +57,9 @@ module Gitlab variables.append(key: 'CI_COMMIT_TIMESTAMP', value: pipeline.git_commit_timestamp.to_s) variables.append(key: 'CI_COMMIT_AUTHOR', value: pipeline.git_author_full_text.to_s) - variables.concat(legacy_predefined_commit_variables) + if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) + variables.concat(legacy_predefined_commit_variables) + end end end strong_memoize_attr :predefined_commit_variables @@ -81,7 +83,9 @@ module Gitlab variables.append(key: 'CI_COMMIT_TAG', value: pipeline.ref) variables.append(key: 'CI_COMMIT_TAG_MESSAGE', value: git_tag.message) - variables.concat(legacy_predefined_commit_tag_variables) + if Feature.disabled?(:ci_remove_legacy_predefined_variables, pipeline.project) + variables.concat(legacy_predefined_commit_tag_variables) + end end end strong_memoize_attr :predefined_commit_tag_variables diff --git a/lib/gitlab/ci/yaml_processor.rb b/lib/gitlab/ci/yaml_processor.rb index 59acfa80258..c69d9218a66 100644 --- a/lib/gitlab/ci/yaml_processor.rb +++ b/lib/gitlab/ci/yaml_processor.rb @@ -94,23 +94,38 @@ module Gitlab end def validate_job_needs!(name, job) - return unless needs = job.dig(:needs, :job) + validate_needs_specification!(name, job.dig(:needs, :job)) - validate_duplicate_needs!(name, needs) + job[:rules]&.each do |rule| + validate_needs_specification!(name, rule.dig(:needs, :job)) + end + end + + def validate_needs_specification!(name, needs) + return unless needs needs.each do |need| - validate_job_dependency!(name, need[:name], 'need') + validate_job_dependency!(name, need[:name], 'need', optional: need[:optional]) end - end - def validate_duplicate_needs!(name, needs) - unless needs.uniq == needs - error!("#{name} has duplicate entries in the needs section.") + duplicated_needs = + needs + .group_by { |need| need[:name] } + .select { |_, items| items.count > 1 } + .keys + + unless duplicated_needs.empty? + error!("#{name} has the following needs duplicated: #{duplicated_needs.join(', ')}.") end end - def validate_job_dependency!(name, dependency, dependency_type = 'dependency') + def validate_job_dependency!(name, dependency, dependency_type = 'dependency', optional: false) unless @jobs[dependency.to_sym] + # Here, we ignore the optional needed job if it is not in the result YAML due to the `include` + # rules. In `lib/gitlab/ci/pipeline/seed/build.rb`, we use `optional` again to ignore the + # optional needed job in case it is excluded from the pipeline due to the job's rules. + return if optional + error!("#{name} job: undefined #{dependency_type}: #{dependency}") end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index d867439b10b..6207b595fc6 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -123,7 +123,8 @@ module Gitlab start_in: job[:start_in], trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, - release: job[:release] + release: job[:release], + publish: job[:publish] }.compact }.compact end diff --git a/lib/gitlab/color.rb b/lib/gitlab/color.rb index 7d9280ddba2..c31c8cb5876 100644 --- a/lib/gitlab/color.rb +++ b/lib/gitlab/color.rb @@ -9,7 +9,7 @@ module Gitlab end module Constants - DARK = Color.new('#333333') + DARK = Color.new('#1F1E24') LIGHT = Color.new('#FFFFFF') COLOR_NAME_TO_HEX = { @@ -194,8 +194,43 @@ module Gitlab PATTERN.match?(@value) end + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L52-55 + def to_srgb(value) + normalized = value / 255.0 + normalized <= 0.03928 ? normalized / 12.92 : ((normalized + 0.055) / 1.055)**2.4 + end + + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L57-64 + def relative_luminance(rgb) + # WCAG 2.1 formula: https://www.w3.org/TR/WCAG21/#dfn-relative-luminance + # - + # WCAG 3.0 will use APAC + # Using APAC would be the ultimate goal, but was dismissed by engineering as of now + # See https://gitlab.com/gitlab-org/gitlab-ui/-/merge_requests/3418#note_1370107090 + (0.2126 * to_srgb(rgb[0])) + (0.7152 * to_srgb(rgb[1])) + (0.0722 * to_srgb(rgb[2])) + end + + # Implementation should match + # https://gitlab.com/gitlab-org/gitlab-ui/-/blob/6245128c7256e3d8db164b92e9580c79d47e9183/src/utils/utils.js#L66-91 def light? - valid? && rgb.sum > 500 + return false unless valid? + + luminance = relative_luminance(rgb) + light_luminance = relative_luminance([255, 255, 255]) + dark_luminance = relative_luminance([31, 30, 36]) + + contrast_light = (light_luminance + 0.05) / (luminance + 0.05) + contrast_dark = (luminance + 0.05) / (dark_luminance + 0.05) + + # Using a threshold contrast of 2.4 instead of 3 + # as this will solve weird color combinations in the mid tones + # + # Note that this is the negated condition from GitLab UI, + # because the GitLab UI implementation returns the text color, + # while this defines whether a background color is light + !(contrast_light >= 2.4 || contrast_light > contrast_dark) end def luminosity diff --git a/lib/gitlab/color_schemes.rb b/lib/gitlab/color_schemes.rb index 3884f5f0428..1ba99e23d65 100644 --- a/lib/gitlab/color_schemes.rb +++ b/lib/gitlab/color_schemes.rb @@ -46,7 +46,7 @@ module Gitlab # # Returns a Scheme def self.default - by_id(1) + by_id(Gitlab::CurrentSettings.default_syntax_highlighting_theme) end # Iterate through each Scheme diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index fad2260d818..28cfb6d8fee 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -356,7 +356,7 @@ module Gitlab ports_size = value.count return if ports_size <= 1 - named_ports = value.select { |e| e.is_a?(Hash) }.map { |e| e[:name] }.compact.map(&:downcase) + named_ports = value.select { |e| e.is_a?(Hash) }.filter_map { |e| e[:name] }.map(&:downcase) if ports_size != named_ports.size record.errors.add(attribute, 'when there is more than one port, a unique name should be added') diff --git a/lib/gitlab/config/loader/multi_doc_yaml.rb b/lib/gitlab/config/loader/multi_doc_yaml.rb index 346adc79896..084d32a85bc 100644 --- a/lib/gitlab/config/loader/multi_doc_yaml.rb +++ b/lib/gitlab/config/loader/multi_doc_yaml.rb @@ -4,59 +4,50 @@ module Gitlab module Config module Loader class MultiDocYaml - TooManyDocumentsError = Class.new(Loader::FormatError) - DataTooLargeError = Class.new(Loader::FormatError) - NotHashError = Class.new(Loader::FormatError) + include Gitlab::Utils::StrongMemoize - MULTI_DOC_DIVIDER = /^---$/.freeze + MULTI_DOC_DIVIDER = /^---\s+/.freeze - def initialize(config, max_documents:, additional_permitted_classes: []) + def initialize(config, max_documents:, additional_permitted_classes: [], reject_empty: false) + @config = config @max_documents = max_documents - @safe_config = load_config(config, additional_permitted_classes) + @additional_permitted_classes = additional_permitted_classes + @reject_empty = reject_empty end - def load! - raise TooManyDocumentsError, 'The parsed YAML has too many documents' if too_many_documents? - raise DataTooLargeError, 'The parsed YAML is too big' if too_big? - raise NotHashError, 'Invalid configuration format' unless all_hashes? - - safe_config.map(&:deep_symbolize_keys) + def valid? + documents.all?(&:valid?) end - private - - attr_reader :safe_config, :max_documents - - def load_config(config, additional_permitted_classes) - config.split(MULTI_DOC_DIVIDER).filter_map do |document| - YAML.safe_load(document, - permitted_classes: [Symbol, *additional_permitted_classes], - permitted_symbols: [], - aliases: true - ) - end - rescue Psych::Exception => e - raise Loader::FormatError, e.message + def load_raw! + documents.map(&:load_raw!) end - def all_hashes? - safe_config.all?(Hash) + def load! + documents.map(&:load!) end - def too_many_documents? - safe_config.count > max_documents - end + private + + attr_reader :config, :max_documents, :additional_permitted_classes, :reject_empty + + # Valid YAML files can start with either a leading delimiter or no delimiter. + # To avoid counting a leading delimiter towards the document limit, + # this method splits the file by one more than the maximum number of permitted documents. + # It then discards the first document if it is blank. + def documents + docs = config + .split(MULTI_DOC_DIVIDER, max_documents_including_leading_delimiter) + .map { |d| Yaml.new(d, additional_permitted_classes: additional_permitted_classes) } - def too_big? - !deep_sizes.all?(&:valid?) + docs.shift if docs.first.blank? + docs.reject!(&:blank?) if reject_empty + docs end + strong_memoize_attr :documents - def deep_sizes - safe_config.map do |config| - Gitlab::Utils::DeepSize.new(config, - max_size: Gitlab::CurrentSettings.current_application_settings.max_yaml_size_bytes, - max_depth: Gitlab::CurrentSettings.current_application_settings.max_yaml_depth) - end + def max_documents_including_leading_delimiter + max_documents + 1 end end end diff --git a/lib/gitlab/config/loader/yaml.rb b/lib/gitlab/config/loader/yaml.rb index 7b87b5b8f97..7138663811e 100644 --- a/lib/gitlab/config/loader/yaml.rb +++ b/lib/gitlab/config/loader/yaml.rb @@ -1,5 +1,9 @@ # frozen_string_literal: true +# NOTE: DO NOT use this class for loading GitLab CI configuration files. +# Instead, use `Gitlab::Ci::Config::Yaml.load!`, which will properly handle +# CI configuration headers. + module Gitlab module Config module Loader @@ -34,6 +38,10 @@ module Gitlab @symbolized_config ||= load_raw!.deep_symbolize_keys end + def blank? + @config.blank? + end + private def hash? diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index ff20833b5be..c95e19940c3 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -21,9 +21,9 @@ module Gitlab { type: 'warning', message: _('Database \'%{database_name}\' is using PostgreSQL %{pg_version_current}, ' \ - 'but PostgreSQL %{pg_version_minimum} is required for this version of GitLab. ' \ - 'Please upgrade your environment to a supported PostgreSQL version, ' \ - 'see %{pg_requirements_url} for details.') % \ + 'but this version of GitLab requires PostgreSQL %{pg_version_minimum}. ' \ + 'Please upgrade your environment to a supported PostgreSQL version. ' \ + 'See %{pg_requirements_url} for details.') % \ { database_name: database_name, pg_version_current: database.version, diff --git a/lib/gitlab/consul/internal.rb b/lib/gitlab/consul/internal.rb index 1994369dee9..c4feac412e4 100644 --- a/lib/gitlab/consul/internal.rb +++ b/lib/gitlab/consul/internal.rb @@ -12,7 +12,7 @@ module Gitlab class << self def api_url Gitlab.config.consul.api_url.to_s.presence if Gitlab.config.consul - rescue Settingslogic::MissingSetting + rescue GitlabSettings::MissingSetting Gitlab::AppLogger.error('Consul api_url is not present in config/gitlab.yml') nil diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index ceca206b084..669c447c09b 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -24,7 +24,7 @@ module Gitlab 'frame_src' => ContentSecurityPolicy::Directives.frame_src, 'img_src' => "'self' data: blob: http: https:", 'manifest_src' => "'self'", - 'media_src' => "'self' data: http: https:", + 'media_src' => "'self' data: blob: http: https:", 'script_src' => ContentSecurityPolicy::Directives.script_src, 'style_src' => ContentSecurityPolicy::Directives.style_src, 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", diff --git a/lib/gitlab/cycle_analytics/stage_summary.rb b/lib/gitlab/cycle_analytics/stage_summary.rb index fdbf068303f..2276f2fc3c6 100644 --- a/lib/gitlab/cycle_analytics/stage_summary.rb +++ b/lib/gitlab/cycle_analytics/stage_summary.rb @@ -45,7 +45,7 @@ module Gitlab end def user_has_sufficient_access? - @project.team.member?(@current_user, Gitlab::Access::REPORTER) + @project.member?(@current_user, Gitlab::Access::REPORTER) end def serialize(summary_object, with_unit: false) diff --git a/lib/gitlab/data_builder/deployment.rb b/lib/gitlab/data_builder/deployment.rb index 7055f64937d..b26f9a61ee1 100644 --- a/lib/gitlab/data_builder/deployment.rb +++ b/lib/gitlab/data_builder/deployment.rb @@ -35,6 +35,7 @@ module Gitlab deployable_id: deployment.deployable_id, deployable_url: deployable_url, environment: deployment.environment.name, + environment_tier: deployment.environment.tier, environment_slug: deployment.environment.slug, environment_external_url: deployment.environment.external_url, project: deployment.project.hook_attrs, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 939eaa377aa..ecb0cc20a64 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -45,8 +45,9 @@ module Gitlab # rubocop: disable CodeReuse/ActiveRecord def preload_builds(pipeline, association) - ActiveRecord::Associations::Preloader.new.preload(pipeline, - { + ActiveRecord::Associations::Preloader.new( + records: [pipeline], + associations: { association => { **::Ci::Pipeline::PROJECT_ROUTE_AND_NAMESPACE_ROUTE, runner: :tags, @@ -56,7 +57,7 @@ module Gitlab ci_stage: [] } } - ) + ).call end # rubocop: enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 756d0afa7e4..4197c87f51f 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -18,7 +18,7 @@ module Gitlab # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements - MINIMUM_POSTGRES_VERSION = 12 + MINIMUM_POSTGRES_VERSION = 13 # https://www.postgresql.org/docs/9.2/static/datatype-numeric.html MAX_INT_VALUE = 2147483647 @@ -35,7 +35,7 @@ module Gitlab MAX_TEXT_SIZE_LIMIT = 1_000_000 # Migrations before this version may have been removed - MIN_SCHEMA_GITLAB_VERSION = '15.0' + MIN_SCHEMA_GITLAB_VERSION = '15.11' # Schema we store dynamically managed partitions in (e.g. for time partitioning) DYNAMIC_PARTITIONS_SCHEMA = :gitlab_partitions_dynamic @@ -51,6 +51,11 @@ module Gitlab FULLY_QUALIFIED_IDENTIFIER = /^\w+\.\w+$/ + ## Database Modes + MODE_SINGLE_DATABASE = "single-database" + MODE_SINGLE_DATABASE_CI_CONNECTION = "single-database-ci-connection" + MODE_MULTIPLE_DATABASES = "multiple-databases" + def self.database_base_models @database_base_models ||= { # Note that we use ActiveRecord::Base here and not ApplicationRecord. @@ -128,12 +133,29 @@ module Gitlab Gitlab::Runtime.max_threads + headroom end + # Database configured. Returns true even if the database is shared def self.has_config?(database_name) ActiveRecord::Base.configurations .configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true) .present? end + # Database configured. Returns false if the database is shared + def self.has_database?(database_name) + db_config = ::Gitlab::Database.database_base_models[database_name]&.connection_db_config + db_config.present? && db_config_share_with(db_config).nil? + end + + def self.database_mode + if !has_config?(CI_DATABASE_NAME) + MODE_SINGLE_DATABASE + elsif has_database?(CI_DATABASE_NAME) + MODE_MULTIPLE_DATABASES + else + MODE_SINGLE_DATABASE_CI_CONNECTION + end + end + class PgUser < ApplicationRecord self.table_name = 'pg_user' self.primary_key = :usename @@ -171,13 +193,12 @@ module Gitlab ███ ███ ██ ██ ██ ██ ██ ████ ██ ██ ████ ██████ ****************************************************************************** - You are using PostgreSQL #{database.version} for the #{name} database, but PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %> - is required for this version of GitLab. + You are using PostgreSQL #{database.version} for the #{name} database, but this version of GitLab requires PostgreSQL >= <%= Gitlab::Database::MINIMUM_POSTGRES_VERSION %>. <% if Rails.env.development? || Rails.env.test? %> If using gitlab-development-kit, please find the relevant steps here: https://gitlab.com/gitlab-org/gitlab-development-kit/-/blob/main/doc/howto/postgresql.md#upgrade-postgresql <% end %> - Please upgrade your environment to a supported PostgreSQL version, see + Please upgrade your environment to a supported PostgreSQL version. See https://docs.gitlab.com/ee/install/requirements.html#database for details. ****************************************************************************** EOS diff --git a/lib/gitlab/database/async_foreign_keys.rb b/lib/gitlab/database/async_constraints.rb index 115ae9ba2e8..026197c7e40 100644 --- a/lib/gitlab/database/async_foreign_keys.rb +++ b/lib/gitlab/database/async_constraints.rb @@ -2,12 +2,12 @@ module Gitlab module Database - module AsyncForeignKeys + module AsyncConstraints DEFAULT_ENTRIES_PER_INVOCATION = 2 def self.validate_pending_entries!(how_many: DEFAULT_ENTRIES_PER_INVOCATION) - PostgresAsyncForeignKeyValidation.ordered.limit(how_many).each do |record| - ForeignKeyValidator.new(record).perform + PostgresAsyncConstraintValidation.ordered.limit(how_many).each do |record| + AsyncConstraints::Validators.for(record).perform end end end diff --git a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb b/lib/gitlab/database/async_constraints/migration_helpers.rb index b8b9fc6d156..8b4d4ecea04 100644 --- a/lib/gitlab/database/async_foreign_keys/migration_helpers.rb +++ b/lib/gitlab/database/async_constraints/migration_helpers.rb @@ -2,7 +2,7 @@ module Gitlab module Database - module AsyncForeignKeys + module AsyncConstraints module MigrationHelpers # Prepares a foreign key for asynchronous validation. # @@ -12,7 +12,7 @@ module Gitlab def prepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? fk_name = name || concurrent_foreign_key_name(table_name, column_name) @@ -20,7 +20,8 @@ module Gitlab raise missing_schema_object_message(table_name, "foreign key", fk_name) end - async_validation = PostgresAsyncForeignKeyValidation + async_validation = PostgresAsyncConstraintValidation + .foreign_key_type .find_or_create_by!(name: fk_name, table_name: table_name) Gitlab::AppLogger.info( @@ -34,17 +35,20 @@ module Gitlab def unprepare_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? fk_name = name || concurrent_foreign_key_name(table_name, column_name) - PostgresAsyncForeignKeyValidation.find_by(name: fk_name).try(&:destroy) + PostgresAsyncConstraintValidation + .foreign_key_type + .find_by(name: fk_name, table_name: table_name) + .try(&:destroy!) end def prepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| prepare_async_foreign_key_validation(partition.identifier, column_name, name: name) @@ -54,17 +58,54 @@ module Gitlab def unprepare_partitioned_async_foreign_key_validation(table_name, column_name = nil, name: nil) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! - return unless async_fk_validation_available? + return unless async_constraint_validation_available? Gitlab::Database::PostgresPartitionedTable.each_partition(table_name) do |partition| unprepare_async_foreign_key_validation(partition.identifier, column_name, name: name) end end + # Prepares a check constraint for asynchronous validation. + # + # Stores the constraint information in the postgres_async_foreign_key_validations + # table to be executed later. + # + def prepare_async_check_constraint_validation(table_name, name:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_constraint_validation_available? + + unless check_constraint_exists?(table_name, name) + raise missing_schema_object_message(table_name, "check constraint", name) + end + + async_validation = PostgresAsyncConstraintValidation + .check_constraint_type + .find_or_create_by!(name: name, table_name: table_name) + + Gitlab::AppLogger.info( + message: 'Prepared check constraint for async validation', + table_name: async_validation.table_name, + constraint_name: async_validation.name) + + async_validation + end + + def unprepare_async_check_constraint_validation(table_name, name:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_constraint_validation_available? + + PostgresAsyncConstraintValidation + .check_constraint_type + .find_by(name: name, table_name: table_name) + .try(&:destroy!) + end + private - def async_fk_validation_available? - connection.table_exists?(:postgres_async_foreign_key_validations) + def async_constraint_validation_available? + PostgresAsyncConstraintValidation.table_available? end end end diff --git a/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb new file mode 100644 index 00000000000..7fb62948119 --- /dev/null +++ b/lib/gitlab/database/async_constraints/postgres_async_constraint_validation.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + class PostgresAsyncConstraintValidation < SharedModel + include QueueErrorHandlingConcern + + self.table_name = 'postgres_async_foreign_key_validations' + + MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH + MAX_LAST_ERROR_LENGTH = 10_000 + + validates :name, presence: true, uniqueness: { scope: :table_name }, length: { maximum: MAX_IDENTIFIER_LENGTH } + validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } + + enum constraint_type: { foreign_key: 0, check_constraint: 1 } + + scope :ordered, -> { order(attempts: :asc, id: :asc) } + scope :foreign_key_type, -> { constraint_type_exists? ? foreign_key : all } + scope :check_constraint_type, -> { check_constraint } + + class << self + def table_available? + connection.table_exists?(table_name) + end + + def constraint_type_exists? + connection.column_exists?(table_name, :constraint_type) + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators.rb b/lib/gitlab/database/async_constraints/validators.rb new file mode 100644 index 00000000000..39792a5ee8e --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + MAPPING = { + foreign_key: Validators::ForeignKey, + check_constraint: Validators::CheckConstraint + }.freeze + + def self.for(record) + MAPPING + .fetch(record.constraint_type.to_sym) + .new(record) + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/base.rb b/lib/gitlab/database/async_constraints/validators/base.rb new file mode 100644 index 00000000000..39a72955d63 --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/base.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class Base + include AsyncDdlExclusiveLeaseGuard + extend ::Gitlab::Utils::Override + + TIMEOUT_PER_ACTION = 1.day + STATEMENT_TIMEOUT = 12.hours + + def initialize(record) + @record = record + end + + def perform + try_obtain_lease do + if constraint_exists? + log_info('Starting to validate constraint') + validate_constraint_with_error_handling + log_info('Finished validating constraint') + else + log_info(skip_log_message) + record.destroy! + end + end + end + + private + + attr_reader :record + + delegate :connection, :name, :table_name, :connection_db_config, to: :record + + def constraint_exists?; end + + def validate_constraint_with_error_handling + validate_constraint + record.destroy! + rescue StandardError => error + record.handle_exception!(error) + + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) + Gitlab::AppLogger.error(message: error.message, **logging_options) + end + + def validate_constraint + set_statement_timeout do + connection.execute(<<~SQL.squish) + ALTER TABLE #{connection.quote_table_name(table_name)} + VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; + SQL + end + end + + def set_statement_timeout + connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) + yield + ensure + connection.execute('RESET statement_timeout') + end + + def lease_timeout + TIMEOUT_PER_ACTION + end + + def log_info(message) + Gitlab::AppLogger.info(message: message, **logging_options) + end + + def skip_log_message + "Skipping #{name} validation since it does not exist. " \ + "The queuing entry will be deleted" + end + + def logging_options + { + class: self.class.name.to_s, + connection_name: database_config_name, + constraint_name: name, + constraint_type: record.constraint_type, + table_name: table_name + } + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/check_constraint.rb b/lib/gitlab/database/async_constraints/validators/check_constraint.rb new file mode 100644 index 00000000000..695ecdebc9f --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/check_constraint.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class CheckConstraint < Base + private + + override :constraint_exists? + def constraint_exists? + Gitlab::Database::Migrations::ConstraintsHelpers + .check_constraint_exists?(table_name, name, connection: connection) + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_constraints/validators/foreign_key.rb b/lib/gitlab/database/async_constraints/validators/foreign_key.rb new file mode 100644 index 00000000000..ff6b807c982 --- /dev/null +++ b/lib/gitlab/database/async_constraints/validators/foreign_key.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module AsyncConstraints + module Validators + class ForeignKey < Base + private + + override :constraint_exists? + def constraint_exists? + Gitlab::Database::PostgresForeignKey + .by_constrained_table_name_or_identifier(table_name) + .by_name(name) + .exists? + end + end + end + end + end +end diff --git a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb b/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb deleted file mode 100644 index 5958c56a45a..00000000000 --- a/lib/gitlab/database/async_foreign_keys/foreign_key_validator.rb +++ /dev/null @@ -1,94 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module AsyncForeignKeys - class ForeignKeyValidator - include AsyncDdlExclusiveLeaseGuard - - TIMEOUT_PER_ACTION = 1.day - STATEMENT_TIMEOUT = 12.hours - - def initialize(async_validation) - @async_validation = async_validation - end - - def perform - try_obtain_lease do - if foreign_key_exists? - log_index_info("Starting to validate foreign key") - validate_foreign_with_error_handling - log_index_info("Finished validating foreign key") - else - log_index_info(skip_log_message) - async_validation.destroy! - end - end - end - - private - - attr_reader :async_validation - - delegate :connection, :name, :table_name, :connection_db_config, to: :async_validation - - def foreign_key_exists? - relation = if table_name =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER - Gitlab::Database::PostgresForeignKey.by_constrained_table_identifier(table_name) - else - Gitlab::Database::PostgresForeignKey.by_constrained_table_name(table_name) - end - - relation.by_name(name).exists? - end - - def validate_foreign_with_error_handling - validate_foreign_key - async_validation.destroy! - rescue StandardError => error - async_validation.handle_exception!(error) - - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error) - Gitlab::AppLogger.error(message: error.message, **logging_options) - end - - def validate_foreign_key - set_statement_timeout do - connection.execute(<<~SQL.squish) - ALTER TABLE #{connection.quote_table_name(table_name)} - VALIDATE CONSTRAINT #{connection.quote_column_name(name)}; - SQL - end - end - - def set_statement_timeout - connection.execute(format("SET statement_timeout TO '%ds'", STATEMENT_TIMEOUT)) - yield - ensure - connection.execute('RESET statement_timeout') - end - - def lease_timeout - TIMEOUT_PER_ACTION - end - - def log_index_info(message) - Gitlab::AppLogger.info(message: message, **logging_options) - end - - def skip_log_message - "Skipping #{name} validation since it does not exist. " \ - "The queuing entry will be deleted" - end - - def logging_options - { - fk_name: name, - table_name: table_name, - class: self.class.name.to_s - } - end - end - end - end -end diff --git a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb b/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb deleted file mode 100644 index de69a3d496f..00000000000 --- a/lib/gitlab/database/async_foreign_keys/postgres_async_foreign_key_validation.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module AsyncForeignKeys - class PostgresAsyncForeignKeyValidation < SharedModel - include QueueErrorHandlingConcern - - self.table_name = 'postgres_async_foreign_key_validations' - - MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH - MAX_LAST_ERROR_LENGTH = 10_000 - - validates :name, presence: true, uniqueness: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } - - scope :ordered, -> { order(attempts: :asc, id: :asc) } - end - end - end -end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index f459c43e0ee..d7128a20a0b 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -77,6 +77,35 @@ module Gitlab async_index end + def prepare_async_index_from_sql(definition) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + return unless async_index_creation_available? + + table_name, index_name = extract_table_and_index_names_from_concurrent_index!(definition) + + if index_name_exists?(table_name, index_name) + Gitlab::AppLogger.warn( + message: 'Index not prepared because it already exists', + table_name: table_name, + index_name: index_name) + + return + end + + async_index = Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec| + rec.table_name = table_name + rec.definition = definition + end + + Gitlab::AppLogger.info( + message: 'Prepared index for async creation', + table_name: async_index.table_name, + index_name: async_index.name) + + async_index + end + # Prepares an index for asynchronous destruction. # # Stores the index information in the postgres_async_indexes table to be removed later. The @@ -110,7 +139,30 @@ module Gitlab end def async_index_creation_available? - connection.table_exists?(:postgres_async_indexes) + table_exists?(:postgres_async_indexes) + end + + private + + delegate :table_exists?, to: :connection, private: true + + def extract_table_and_index_names_from_concurrent_index!(definition) + statement = index_statement_from!(definition) + + raise 'Index statement not found!' unless statement + raise 'Index must be created concurrently!' unless statement.concurrent + raise 'Table does not exist!' unless table_exists?(statement.relation.relname) + + [statement.relation.relname, statement.idxname] + end + + # This raises `PgQuery::ParseError` if the given statement + # is syntactically incorrect, therefore, validates that the + # index definition is correct. + def index_statement_from!(definition) + parsed_query = PgQuery.parse(definition) + + parsed_query.tree.stmts[0].stmt.index_stmt end end end diff --git a/lib/gitlab/database/background_migration/batch_optimizer.rb b/lib/gitlab/database/background_migration/batch_optimizer.rb index c8fdf8281cd..9eb456f6e2e 100644 --- a/lib/gitlab/database/background_migration/batch_optimizer.rb +++ b/lib/gitlab/database/background_migration/batch_optimizer.rb @@ -43,11 +43,14 @@ module Gitlab def optimize! return unless Feature.enabled?(:optimize_batched_migrations, type: :ops) - if multiplier = batch_size_multiplier - max_batch = migration.max_batch_size || MAX_BATCH_SIZE - migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(MIN_BATCH_SIZE, max_batch) - migration.save! - end + multiplier = batch_size_multiplier + return if multiplier.nil? + + max_batch = migration.max_batch_size || MAX_BATCH_SIZE + min_batch = [max_batch, MIN_BATCH_SIZE].min + + migration.batch_size = (migration.batch_size * multiplier).to_i.clamp(min_batch, max_batch) + migration.save! end private diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 6b7ff308c7e..523ab2a9f27 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -4,6 +4,7 @@ module Gitlab module Database module BackgroundMigration SplitAndRetryError = Class.new(StandardError) + ReduceSubBatchSizeError = Class.new(StandardError) class BatchedJob < SharedModel include EachBatch @@ -12,6 +13,9 @@ module Gitlab self.table_name = :batched_background_migration_jobs MAX_ATTEMPTS = 3 + MIN_BATCH_SIZE = 1 + SUB_BATCH_SIZE_REDUCE_FACTOR = 0.75 + SUB_BATCH_SIZE_THRESHOLD = 65 STUCK_JOBS_TIMEOUT = 1.hour.freeze TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout, @@ -59,12 +63,12 @@ module Gitlab end after_transition any => :failed do |job, transition| - error_hash = transition.args.find { |arg| arg[:error].present? } + exception, from_sub_batch = job.class.extract_transition_options(transition.args) - exception = error_hash&.fetch(:error) + job.reduce_sub_batch_size! if from_sub_batch && job.can_reduce_sub_batch_size? job.split_and_retry! if job.can_split?(exception) - rescue SplitAndRetryError => error + rescue SplitAndRetryError, ReduceSubBatchSizeError => error Gitlab::AppLogger.error( message: error.message, batched_job_id: job.id, @@ -75,9 +79,7 @@ module Gitlab end after_transition do |job, transition| - error_hash = transition.args.find { |arg| arg[:error].present? } - - exception = error_hash&.fetch(:error) + exception, _ = job.class.extract_transition_options(transition.args) job.batched_job_transition_logs.create(previous_status: transition.from, next_status: transition.to, exception_class: exception&.class, exception_message: exception&.message) @@ -100,7 +102,16 @@ module Gitlab delegate :job_class, :table_name, :column_name, :job_arguments, :job_class_name, to: :batched_migration, prefix: :migration - attribute :pause_ms, :integer, default: 100 + def self.extract_transition_options(args) + error_hash = args.find { |arg| arg[:error].present? } + + return [] unless error_hash + + exception = error_hash.fetch(:error) + from_sub_batch = error_hash[:from_sub_batch] + + [exception, from_sub_batch] + end def time_efficiency return unless succeeded? @@ -113,10 +124,13 @@ module Gitlab end def can_split?(exception) - attempts >= MAX_ATTEMPTS && - exception&.class&.in?(TIMEOUT_EXCEPTIONS) && - batch_size > sub_batch_size && - batch_size > 1 + return if still_retryable? + + exception.class.in?(TIMEOUT_EXCEPTIONS) && within_batch_size_boundaries? + end + + def can_reduce_sub_batch_size? + still_retryable? && within_batch_size_boundaries? end def split_and_retry! @@ -165,6 +179,51 @@ module Gitlab end end end + + # It reduces the size of +sub_batch_size+ by 25% + def reduce_sub_batch_size! + raise ReduceSubBatchSizeError, 'Only sub_batch_size of failed jobs can be reduced' unless failed? + + return if sub_batch_exceeds_threshold? + + with_lock do + actual_sub_batch_size = sub_batch_size + reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i.clamp(1, batch_size) + + update!(sub_batch_size: reduced_sub_batch_size) + + Gitlab::AppLogger.warn( + message: 'Sub batch size reduced due to timeout', + batched_job_id: id, + sub_batch_size: actual_sub_batch_size, + reduced_sub_batch_size: reduced_sub_batch_size, + attempts: attempts, + batched_migration_id: batched_migration.id, + job_class_name: migration_job_class_name, + job_arguments: migration_job_arguments + ) + end + end + + def still_retryable? + attempts < MAX_ATTEMPTS + end + + def within_batch_size_boundaries? + batch_size > MIN_BATCH_SIZE && batch_size > sub_batch_size + end + + # It doesn't allow sub-batch size to be reduced lower than the threshold + # + # @info It will prevent the next iteration to reduce the +sub_batch_size+ lower + # than the +SUB_BATCH_SIZE_THRESHOLD+ or 65% of its original size. + def sub_batch_exceeds_threshold? + initial_sub_batch_size = batched_migration.sub_batch_size + reduced_sub_batch_size = (sub_batch_size * SUB_BATCH_SIZE_REDUCE_FACTOR).to_i + diff = initial_sub_batch_size - reduced_sub_batch_size + + (1.0 * diff / initial_sub_batch_size * 100).round(2) > SUB_BATCH_SIZE_THRESHOLD + end end end end diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 61a660ad14c..a883996a5c5 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -25,6 +25,7 @@ module Gitlab scope :queue_order, -> { order(id: :asc) } scope :queued, -> { with_statuses(:active, :paused) } + scope :finalizing, -> { with_status(:finalizing) } scope :ordered_by_created_at_desc, -> { order(created_at: :desc) } # on_hold_until is a temporary runtime status which puts execution "on hold" @@ -83,8 +84,6 @@ module Gitlab end end - attribute :pause_ms, :integer, default: 100 - def self.valid_status state_machine.states.map(&:name) end @@ -221,7 +220,7 @@ module Gitlab end def health_context - HealthStatus::Context.new(connection, [table_name]) + HealthStatus::Context.new(connection, [table_name], gitlab_schema.to_sym) end def hold!(until_time: 10.minutes.from_now) diff --git a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb index f1fc3efae9e..8fdaa685ba9 100644 --- a/lib/gitlab/database/background_migration/batched_migration_wrapper.rb +++ b/lib/gitlab/database/background_migration/batched_migration_wrapper.rb @@ -15,13 +15,21 @@ module Gitlab # when starting and finishing execution, and optionally saves batch_metrics # the migration provides, if any are given. # - # The job's batch_metrics are serialized to JSON for storage. + # @info The job's batch_metrics are serialized to JSON for storage. + # + # @info Track exceptions that could happen when processing sub-batches + # through +Gitlab::BackgroundMigration::SubBatchTimeoutException+ def perform(batch_tracking_record) start_tracking_execution(batch_tracking_record) execute_batch(batch_tracking_record) batch_tracking_record.succeed! + rescue SubBatchTimeoutError => exception + caused_by = exception.caused_by + batch_tracking_record.failure!(error: caused_by, from_sub_batch: true) + + raise caused_by rescue Exception => error # rubocop:disable Lint/RescueException batch_tracking_record.failure!(error: error) @@ -67,7 +75,8 @@ module Gitlab sub_batch_size: tracking_record.sub_batch_size, pause_ms: tracking_record.pause_ms, job_arguments: tracking_record.migration_job_arguments, - connection: connection) + connection: connection, + sub_batch_exception: ::Gitlab::Database::BackgroundMigration::SubBatchTimeoutError) job_instance.perform diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb index 506d2996ad5..96905fd0bc5 100644 --- a/lib/gitlab/database/background_migration/health_status.rb +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -6,11 +6,12 @@ module Gitlab module HealthStatus DEFAULT_INIDICATORS = [ Indicators::AutovacuumActiveOnTable, - Indicators::WriteAheadLog + Indicators::WriteAheadLog, + Indicators::PatroniApdex ].freeze # Rather than passing along the migration, we use a more explicitly defined context - Context = Struct.new(:connection, :tables) + Context = Struct.new(:connection, :tables, :gitlab_schema) def self.evaluate(migration, indicators = DEFAULT_INIDICATORS) indicators.map do |indicator| @@ -30,9 +31,12 @@ module Gitlab end def self.log_signal(signal, migration) - Gitlab::AppLogger.info( - message: "#{migration} signaled: #{signal}", - migration_id: migration.id + Gitlab::BackgroundMigration::Logger.info( + migration_id: migration.id, + health_status_indicator: signal.indicator_class.to_s, + indicator_signal: signal.short_name, + signal_reason: signal.reason, + message: "#{migration} signaled: #{signal}" ) end end diff --git a/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb new file mode 100644 index 00000000000..0dd6dd5c2a4 --- /dev/null +++ b/lib/gitlab/database/background_migration/health_status/indicators/patroni_apdex.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + module HealthStatus + module Indicators + class PatroniApdex + include Gitlab::Utils::StrongMemoize + + def initialize(context) + @context = context + end + + def evaluate + return Signals::NotAvailable.new(self.class, reason: 'indicator disabled') unless enabled? + + connection_error_message = fetch_connection_error_message + return unknown_signal(connection_error_message) if connection_error_message.present? + + apdex_sli = fetch_sli(apdex_sli_query) + return unknown_signal('Patroni service apdex can not be calculated') unless apdex_sli.present? + + if apdex_sli.to_f > apdex_slo.to_f + Signals::Normal.new(self.class, reason: 'Patroni service apdex is above SLO') + else + Signals::Stop.new(self.class, reason: 'Patroni service apdex is below SLO') + end + end + + private + + attr_reader :context + + def enabled? + Feature.enabled?(:batched_migrations_health_status_patroni_apdex, type: :ops) + end + + def unknown_signal(reason) + Signals::Unknown.new(self.class, reason: reason) + end + + def fetch_connection_error_message + return 'Patroni Apdex Settings not configured' unless database_apdex_settings.present? + return 'Prometheus client is not ready' unless client.ready? + return 'Apdex SLI query is not configured' unless apdex_sli_query + return 'Apdex SLO is not configured' unless apdex_slo + end + + def client + @client ||= Gitlab::PrometheusClient.new( + database_apdex_settings[:prometheus_api_url], + allow_local_requests: true, + verify: true + ) + end + + def database_apdex_settings + @database_apdex_settings ||= Gitlab::CurrentSettings.database_apdex_settings&.with_indifferent_access + end + + def apdex_sli_query + { + gitlab_main: database_apdex_settings[:apdex_sli_query][:main], + gitlab_ci: database_apdex_settings[:apdex_sli_query][:ci] + }.fetch(context.gitlab_schema.to_sym) + end + strong_memoize_attr :apdex_sli_query + + def apdex_slo + { + gitlab_main: database_apdex_settings[:apdex_slo][:main], + gitlab_ci: database_apdex_settings[:apdex_slo][:ci] + }.fetch(context.gitlab_schema.to_sym) + end + strong_memoize_attr :apdex_slo + + def fetch_sli(query) + response = client.query(query) + metric = response&.first || {} + value = metric.fetch('value', []) + + Array.wrap(value).second + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/background_migration/health_status/signals.rb b/lib/gitlab/database/background_migration/health_status/signals.rb index be741a9d91b..534c4330cf2 100644 --- a/lib/gitlab/database/background_migration/health_status/signals.rb +++ b/lib/gitlab/database/background_migration/health_status/signals.rb @@ -28,8 +28,6 @@ module Gitlab end # :nocov: - private - def short_name self.class.name.demodulize end diff --git a/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb new file mode 100644 index 00000000000..815dff11e1f --- /dev/null +++ b/lib/gitlab/database/background_migration/sub_batch_timeout_error.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module BackgroundMigration + class SubBatchTimeoutError < StandardError + def initialize(caused_by) + @caused_by = caused_by + + super(caused_by) + end + + attr_reader :caused_by + end + end + end +end diff --git a/lib/gitlab/database/background_migration_job.rb b/lib/gitlab/database/background_migration_job.rb index c0e3016fd3d..5141dd05e4e 100644 --- a/lib/gitlab/database/background_migration_job.rb +++ b/lib/gitlab/database/background_migration_job.rb @@ -13,10 +13,6 @@ module Gitlab for_migration_class(class_name).where('arguments = ?', arguments.to_json) # rubocop:disable Rails/WhereEquals end - scope :for_partitioning_migration, -> (class_name, table_name) do - for_migration_class(class_name).where('arguments ->> 2 = ?', table_name) - end - enum status: { pending: 0, succeeded: 1 diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 7a064fb4005..7249cb3e73b 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -27,7 +27,7 @@ # batch_sum(User, :sign_in_count) # batch_sum(Issue.group(:state_id), :weight)) # batch_average(Ci::Pipeline, :duration) -# batch_average(MergeTrain.group(:status), :duration) +# batch_average(MergeTrains::Car.group(:status), :duration) module Gitlab module Database module BatchCount diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 2deb89a0b84..83edf77f37e 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -17,7 +17,7 @@ module Gitlab klass end - def each_batch(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE) + def each_batch(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE, **opts) if transaction_open? raise <<~MSG.squish each_batch should not run inside a transaction, you can disable @@ -26,13 +26,21 @@ module Gitlab MSG end - scope.call(define_batchable_model(table_name, connection: connection)) - .each_batch(of: of) { |batch| yield batch } + opts.select! { |k, _| [:column].include? k } + + batchable_model = define_batchable_model(table_name, connection: connection) + + scope.call(batchable_model) + .each_batch(of: of, **opts) { |batch| yield batch, batchable_model } end - def each_batch_range(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE) - each_batch(table_name, connection: connection, scope: scope, of: of) do |batch| - yield batch.pick('MIN(id), MAX(id)') + def each_batch_range(table_name, connection:, scope: ->(table) { table.all }, of: BATCH_SIZE, **opts) + opts.select! { |k, _| [:column].include? k } + + each_batch(table_name, connection: connection, scope: scope, of: of, **opts) do |batch, batchable_model| + column = opts.fetch(:column, batchable_model.primary_key) + + yield batch.pick("MIN(#{column}), MAX(#{column})") end end end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 38558512b6a..4394c089b22 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -19,11 +19,12 @@ module Gitlab DICTIONARY_PATH = 'db/docs/' - def self.table_schemas(tables, undefined: true) - tables.map { |table| table_schema(table, undefined: undefined) }.to_set + def self.table_schemas!(tables) + tables.map { |table| table_schema!(table) }.to_set end - def self.table_schema(name, undefined: true) + # rubocop:disable Metrics/CyclomaticComplexity + def self.table_schema(name) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` # Most of names do not have schemas, ensure that this is table @@ -45,6 +46,11 @@ module Gitlab return gitlab_schema end + # Partitions that belong to the CI domain + if table_name.start_with?('ci_') && gitlab_schema = views_and_tables_to_schema["p_#{table_name}"] + return gitlab_schema + end + # All tables from `information_schema.` are marked as `internal` return :gitlab_internal if schema_name == 'information_schema' @@ -52,6 +58,8 @@ module Gitlab return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + return :gitlab_embedding if table_name.start_with?('_test_gitlab_embedding_') + return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') # All tables that start with `_test_` without a following schema are shared and ignored @@ -60,9 +68,14 @@ module Gitlab # All `pg_` tables are marked as `internal` return :gitlab_internal if table_name.start_with?('pg_') - # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database - undefined ? :"undefined_#{table_name}" : nil + # Sometimes the name of an index can be interpreted as a table's name. + # For eg, if we execute "ALTER INDEX my_index..", my_index is interpreted as a table name. + # In such cases, we should return the schema of the database table actually + # holding that index. + index_name = table_name + derive_schema_from_index(index_name) end + # rubocop:enable Metrics/CyclomaticComplexity def self.dictionary_path_globs [Rails.root.join(DICTIONARY_PATH, '*.yml')] @@ -85,10 +98,13 @@ module Gitlab end def self.table_schema!(name) - self.table_schema(name, undefined: false) || raise( + # rubocop:disable Gitlab/DocUrl + self.table_schema(name) || raise( UnknownSchemaError, - "Could not find gitlab schema for table #{name}: Any new tables must be added to the database dictionary" + "Could not find gitlab schema for table #{name}: Any new or deleted tables must be added to the database dictionary " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" ) + # rubocop:enable Gitlab/DocUrl end def self.deleted_views_and_tables_to_schema @@ -115,12 +131,31 @@ module Gitlab @schema_names ||= self.views_and_tables_to_schema.values.to_set end + private_class_method def self.derive_schema_from_index(index_name) + index = Gitlab::Database::PostgresIndex.find_by(name: index_name, + schema: ApplicationRecord.connection.current_schema) + + return unless index + + table_schema(index.tablename) + end + private_class_method def self.build_dictionary(path_globs) Dir.glob(path_globs).each_with_object({}) do |file_path, dic| data = YAML.load_file(file_path) key_name = data['table_name'] || data['view_name'] + # rubocop:disable Gitlab/DocUrl + if data['gitlab_schema'].nil? + raise( + UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ + "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" + ) + end + # rubocop:enable Gitlab/DocUrl + dic[key_name] = data['gitlab_schema'].to_sym end end diff --git a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb index 7164976ff73..fab691117ad 100644 --- a/lib/gitlab/database/load_balancing/action_cable_callbacks.rb +++ b/lib/gitlab/database/load_balancing/action_cable_callbacks.rb @@ -6,14 +6,10 @@ module Gitlab module ActionCableCallbacks def self.install ::ActionCable::Server::Worker.set_callback :work, :around, &wrapper - ::ActionCable::Channel::Base.set_callback :subscribe, :around, &wrapper - ::ActionCable::Channel::Base.set_callback :unsubscribe, :around, &wrapper end def self.wrapper lambda do |_, inner| - ::Gitlab::Database::LoadBalancing::Session.current.use_primary! - inner.call ensure ::Gitlab::Database::LoadBalancing.release_hosts diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 622e310ead3..02f14e020c1 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -62,6 +62,13 @@ module Gitlab end end + def schema_cache(*args, **kwargs, &block) + # Ignore primary stickiness for schema_cache queries and always use replicas + @load_balancer.read do |connection| + connection.public_send(:schema_cache, *args, **kwargs, &block) + end + end + def transaction(*args, **kwargs, &block) if current_session.fallback_to_replicas_for_ambiguous_queries? track_read_only_transaction! diff --git a/lib/gitlab/database/load_balancing/logger.rb b/lib/gitlab/database/load_balancing/logger.rb index ee67ffcc99c..df6ae47bb84 100644 --- a/lib/gitlab/database/load_balancing/logger.rb +++ b/lib/gitlab/database/load_balancing/logger.rb @@ -4,6 +4,8 @@ module Gitlab module Database module LoadBalancing class Logger < ::Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'database_load_balancing' end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 5059b3b5c93..57a588db8a8 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -103,6 +103,15 @@ module Gitlab # Slightly randomize the retry delay so that, in the case of a total # dns outage, all starting services do not pressure the dns server at the same time. sleep(rand(RETRY_DELAY_RANGE)) + rescue Exception => error # rubocop:disable Lint/RescueException + # All exceptions are logged to find any pattern and solve https://gitlab.com/gitlab-org/gitlab/-/issues/364370 + # This will be removed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120173 + Gitlab::Database::LoadBalancing::Logger.error( + event: :service_discovery_unexpected_exception, + message: "Service discovery encountered an uncaught error: #{error.message}" + ) + + raise end interval diff --git a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb index 619f11ae890..346d951414a 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_client_middleware.rb @@ -5,13 +5,16 @@ module Gitlab module LoadBalancing class SidekiqClientMiddleware include Gitlab::Utils::StrongMemoize + include WalTrackingSender def call(worker_class, job, _queue, _redis_pool) # Mailers can't be constantized worker_class = worker_class.to_s.safe_constantize + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped'].to_s.safe_constantize || worker_class - if load_balancing_enabled?(worker_class) - job['worker_data_consistency'] = worker_class.get_data_consistency + if load_balancing_enabled?(resolved_class) + job['worker_data_consistency'] = resolved_class.get_data_consistency set_data_consistency_locations!(job) unless job['wal_locations'] else job['worker_data_consistency'] = ::WorkerAttributes::DEFAULT_DATA_CONSISTENCY @@ -24,21 +27,13 @@ module Gitlab def load_balancing_enabled?(worker_class) worker_class && - worker_class.include?(::ApplicationWorker) && + worker_class.include?(::WorkerAttributes) && worker_class.utilizes_load_balancing_capabilities? && worker_class.get_data_consistency_feature_flag_enabled? end def set_data_consistency_locations!(job) - locations = {} - - ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| - if (location = wal_location_for(lb)) - locations[lb.name] = location - end - end - - job['wal_locations'] = locations + job['wal_locations'] = wal_locations_by_db_name job['wal_location_source'] = wal_location_source end @@ -50,17 +45,6 @@ module Gitlab end end - def wal_location_for(load_balancer) - # When only using the primary there's no need for any WAL queries. - return if load_balancer.primary_only? - - if uses_primary? - load_balancer.primary_write_location - else - load_balancer.host&.database_replica_location || load_balancer.primary_write_location - end - end - def uses_primary? ::Gitlab::Database::LoadBalancing::Session.current.use_primary? end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index f7b8d2514ba..1cb91a5c45b 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -4,21 +4,24 @@ module Gitlab module Database module LoadBalancing class SidekiqServerMiddleware + include WalTrackingReceiver + JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError) - MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 + REPLICA_WAIT_SLEEP_SECONDS = 0.5 def call(worker, job, _queue) - worker_class = worker.class - strategy = select_load_balancing_strategy(worker_class, job) + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped']&.safe_constantize || worker.class + strategy = select_load_balancing_strategy(resolved_class, job) job['load_balancing_strategy'] = strategy.to_s if use_primary?(strategy) ::Gitlab::Database::LoadBalancing::Session.current.use_primary! elsif strategy == :retry - raise JobReplicaNotUpToDate, "Sidekiq job #{worker_class} JID-#{job['jid']} couldn't use the replica."\ - " Replica was not up to date." + raise JobReplicaNotUpToDate, "Sidekiq job #{resolved_class} JID-#{job['jid']} couldn't use the replica."\ + " Replica was not up to date." else # this means we selected an up-to-date replica, but there is nothing to do in this case. end @@ -49,7 +52,10 @@ module Gitlab # Happy case: we can read from a replica. return replica_strategy(worker_class, job) if databases_in_sync?(wal_locations) - sleep_if_needed(job) + 3.times do + sleep REPLICA_WAIT_SLEEP_SECONDS + break if databases_in_sync?(wal_locations) + end if databases_in_sync?(wal_locations) replica_strategy(worker_class, job) @@ -62,24 +68,18 @@ module Gitlab end end - def sleep_if_needed(job) - remaining_delay = MINIMUM_DELAY_INTERVAL_SECONDS - (Time.current.to_f - job['created_at'].to_f) - - sleep remaining_delay if remaining_delay > 0 && remaining_delay < MINIMUM_DELAY_INTERVAL_SECONDS - end - def get_wal_locations(job) job['dedup_wal_locations'] || job['wal_locations'] end def load_balancing_available?(worker_class) - worker_class.include?(::ApplicationWorker) && + worker_class.include?(::WorkerAttributes) && worker_class.utilizes_load_balancing_capabilities? && worker_class.get_data_consistency_feature_flag_enabled? end def can_retry?(worker_class, job) - worker_class.get_data_consistency == :delayed && not_yet_retried?(job) + worker_class.get_data_consistency == :delayed && not_yet_requeued?(job) end def replica_strategy(worker_class, job) @@ -87,27 +87,14 @@ module Gitlab end def retried_before?(worker_class, job) - worker_class.get_data_consistency == :delayed && !not_yet_retried?(job) + worker_class.get_data_consistency == :delayed && !not_yet_requeued?(job) end - def not_yet_retried?(job) + def not_yet_requeued?(job) # if `retry_count` is `nil` it indicates that this job was never retried # the `0` indicates that this is a first retry job['retry_count'].nil? end - - def databases_in_sync?(wal_locations) - ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| - if (location = wal_locations.with_indifferent_access[lb.name]) - lb.select_up_to_date_host(location) - else - # If there's no entry for a load balancer it means the Sidekiq - # job doesn't care for it. In this case we'll treat the load - # balancer as being in sync. - true - end - end - end end end end diff --git a/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb new file mode 100644 index 00000000000..ef9bee42277 --- /dev/null +++ b/lib/gitlab/database/load_balancing/wal_tracking_receiver.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module WalTrackingReceiver + # NOTE: If there's no entry for a load balancer or no WAL locations were passed + # we assume the sender does not care about LB and we assume nodes are in-sync. + def databases_in_sync?(wal_locations) + return true unless wal_locations.present? + + ::Gitlab::Database::LoadBalancing.each_load_balancer.all? do |lb| + if (location = wal_locations.with_indifferent_access[lb.name]) + lb.select_up_to_date_host(location) + else + true + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/wal_tracking_sender.rb b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb new file mode 100644 index 00000000000..ed0af58d064 --- /dev/null +++ b/lib/gitlab/database/load_balancing/wal_tracking_sender.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + module WalTrackingSender + def wal_locations_by_db_name + {}.tap do |locations| + ::Gitlab::Database::LoadBalancing.each_load_balancer do |lb| + if (location = wal_location_for(lb)) + locations[lb.name] = location + end + end + end + end + + def wal_location_for(load_balancer) + # When only using the primary there's no need for any WAL queries. + return if load_balancer.primary_only? + + if Session.current.use_primary? + load_balancer.primary_write_location + else + load_balancer.host&.database_replica_location || load_balancer.primary_write_location + end + end + end + end + end +end diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index 83884e89d6e..43e71e6bda2 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -10,6 +10,8 @@ module Gitlab # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us EXPECTED_TRIGGER_RECORD_COUNT = 3 + # table_name can include schema name as a prefix. For example: 'gitlab_partitions_static.events_03', + # otherwise, it will default to current used schema, for example 'public'. def initialize(table_name:, connection:, database_name:, with_retries: true, logger: nil, dry_run: false) @table_name = table_name @connection = connection @@ -19,7 +21,7 @@ module Gitlab @with_retries = with_retries @table_name_without_schema = ActiveRecord::ConnectionAdapters::PostgreSQL::Utils - .extract_schema_qualified_name(table_name) + .extract_schema_qualified_name(table_name.to_s) .identifier end @@ -36,7 +38,7 @@ module Gitlab def lock_writes if table_locked_for_writes? logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" - return + return result_hash(action: 'skipped') end logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow) @@ -48,6 +50,8 @@ module Gitlab SQL execute_sql_statement(sql_statement) + + result_hash(action: 'locked') end def unlock_writes @@ -57,6 +61,8 @@ module Gitlab SQL execute_sql_statement(sql_statement) + + result_hash(action: 'unlocked') end private @@ -111,6 +117,10 @@ module Gitlab def write_trigger_name "gitlab_schema_write_trigger_for_#{table_name_without_schema}" end + + def result_hash(action:) + { action: action, database: database_name, table: table_name, dry_run: dry_run } + end end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index 4d38920f571..fbb71c1ccfd 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -53,6 +53,7 @@ module Gitlab class V2_1 < V2_0 # rubocop:disable Naming/ClassAndModuleCamelCase include Gitlab::Database::MigrationHelpers::AutomaticLockWritesOnTables + include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers end def self.[](version) diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 9b041c18da4..291f483e6e4 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -14,7 +14,8 @@ module Gitlab include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers - include AsyncForeignKeys::MigrationHelpers + include AsyncConstraints::MigrationHelpers + include WraparoundVacuumHelpers def define_batchable_model(table_name, connection: self.connection) super(table_name, connection: connection) @@ -79,63 +80,6 @@ module Gitlab end end - # @deprecated Use `create_table` in V2 instead - # - # Creates a new table, optionally allowing the caller to add check constraints to the table. - # Aside from that addition, this method should behave identically to Rails' `create_table` method. - # - # Example: - # - # create_table_with_constraints :some_table do |t| - # t.integer :thing, null: false - # t.text :other_thing - # - # t.check_constraint :thing_is_not_null, 'thing IS NOT NULL' - # t.text_limit :other_thing, 255 - # end - # - # See Rails' `create_table` for more info on the available arguments. - def create_table_with_constraints(table_name, **options, &block) - helper_context = self - - with_lock_retries do - check_constraints = [] - - create_table(table_name, **options) do |t| - t.define_singleton_method(:check_constraint) do |name, definition| - helper_context.send(:validate_check_constraint_name!, name) # rubocop:disable GitlabSecurity/PublicSend - - check_constraints << { name: name, definition: definition } - end - - t.define_singleton_method(:text_limit) do |column_name, limit, name: nil| - # rubocop:disable GitlabSecurity/PublicSend - name = helper_context.send(:text_limit_name, table_name, column_name, name: name) - helper_context.send(:validate_check_constraint_name!, name) - # rubocop:enable GitlabSecurity/PublicSend - - column_name = helper_context.quote_column_name(column_name) - definition = "char_length(#{column_name}) <= #{limit}" - - check_constraints << { name: name, definition: definition } - end - - t.instance_eval(&block) unless block.nil? - end - - next if check_constraints.empty? - - constraint_clauses = check_constraints.map do |constraint| - "ADD CONSTRAINT #{quote_table_name(constraint[:name])} CHECK (#{constraint[:definition]})" - end - - execute(<<~SQL) - ALTER TABLE #{quote_table_name(table_name)} - #{constraint_clauses.join(",\n")} - SQL - end - end - # Creates a new index, concurrently # # Example: @@ -292,23 +236,34 @@ module Gitlab # order of the ALTER TABLE. This can be useful in situations where the foreign # key creation could deadlock with another process. # - # rubocop: disable Metrics/ParameterLists - def add_concurrent_foreign_key(source, target, column:, on_delete: :cascade, on_update: nil, target_column: :id, name: nil, validate: true, reverse_lock_order: false) + def add_concurrent_foreign_key(source, target, column:, **options) + options.reverse_merge!({ + on_delete: :cascade, + on_update: nil, + target_column: :id, + validate: true, + reverse_lock_order: false, + allow_partitioned: false, + column: column + }) + # Transactions would result in ALTER TABLE locks being held for the # duration of the transaction, defeating the purpose of this method. if transaction_open? raise 'add_concurrent_foreign_key can not be run inside a transaction' end - options = { - column: column, - on_delete: on_delete, - on_update: on_update, - name: name.presence || concurrent_foreign_key_name(source, column), - primary_key: target_column - } + if !options.delete(:allow_partitioned) && table_partitioned?(source) + raise ArgumentError, 'add_concurrent_foreign_key can not be used on a partitioned ' \ + 'table. Please use add_concurrent_partitioned_foreign_key on the partitioned table ' \ + 'as we need to create foreign keys on each partition and a FK on the parent table' + end - if foreign_key_exists?(source, target, **options) + options[:name] ||= concurrent_foreign_key_name(source, column) + options[:primary_key] = options[:target_column] + check_options = options.slice(:column, :on_delete, :on_update, :name, :primary_key) + + if foreign_key_exists?(source, target, **check_options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ "source: #{source}, target: #{target}, column: #{options[:column]}, "\ @@ -317,23 +272,7 @@ module Gitlab Gitlab::AppLogger.warn warning_message else - # Using NOT VALID allows us to create a key without immediately - # validating it. This means we keep the ALTER TABLE lock only for a - # short period of time. The key _is_ enforced for any newly created - # data. - - with_lock_retries do - execute("LOCK TABLE #{target}, #{source} IN SHARE ROW EXCLUSIVE MODE") if reverse_lock_order - execute <<-EOF.strip_heredoc - ALTER TABLE #{source} - ADD CONSTRAINT #{options[:name]} - FOREIGN KEY (#{multiple_columns(options[:column])}) - REFERENCES #{target} (#{multiple_columns(target_column)}) - #{on_update_statement(options[:on_update])} - #{on_delete_statement(options[:on_delete])} - NOT VALID; - EOF - end + execute_add_concurrent_foreign_key(source, target, options) end # Validate the existing constraint. This can potentially take a very @@ -345,13 +284,12 @@ module Gitlab # # Note this is a no-op in case the constraint is VALID already - if validate + if options[:validate] disable_statement_timeout do execute("ALTER TABLE #{source} VALIDATE CONSTRAINT #{options[:name]};") end end end - # rubocop: enable Metrics/ParameterLists def validate_foreign_key(source, column, name: nil) fk_name = name || concurrent_foreign_key_name(source, column) @@ -379,7 +317,14 @@ module Gitlab end end - fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name(source) + # Since we may be migrating in one go from a previous version without + # `constrained_table_name` then we may see that this column exists + # (as above) but the schema cache is still outdated for the model. + unless Gitlab::Database::PostgresForeignKey.column_names.include?('constrained_table_name') + Gitlab::Database::PostgresForeignKey.reset_column_information + end + + fks = Gitlab::Database::PostgresForeignKey.by_constrained_table_name_or_identifier(source) fks = fks.by_referenced_table_name(target) if target fks = fks.by_name(options[:name]) if options[:name] @@ -1239,6 +1184,12 @@ into similar problems in the future (e.g. when new tables are created). end end + def table_partitioned?(table_name) + Gitlab::Database::PostgresPartitionedTable + .find_by_name_in_current_schema(table_name) + .present? + end + private def multiple_columns(columns, separator: ', ') @@ -1354,6 +1305,33 @@ into similar problems in the future (e.g. when new tables are created). Must end with `_at`} MESSAGE end + + def execute_add_concurrent_foreign_key(source, target, options) + # Using NOT VALID allows us to create a key without immediately + # validating it. This means we keep the ALTER TABLE lock only for a + # short period of time. The key _is_ enforced for any newly created + # data. + not_valid = 'NOT VALID' + lock_mode = 'SHARE ROW EXCLUSIVE' + + if table_partitioned?(source) + not_valid = '' + lock_mode = 'ACCESS EXCLUSIVE' + end + + with_lock_retries do + execute("LOCK TABLE #{target}, #{source} IN #{lock_mode} MODE") if options[:reverse_lock_order] + execute(<<~SQL.squish) + ALTER TABLE #{source} + ADD CONSTRAINT #{options[:name]} + FOREIGN KEY (#{multiple_columns(options[:column])}) + REFERENCES #{target} (#{multiple_columns(options[:target_column])}) + #{on_update_statement(options[:on_update])} + #{on_delete_statement(options[:on_delete])} + #{not_valid}; + SQL + end + end end end end diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb index c59139344ea..55c4fd6a7af 100644 --- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -44,16 +44,7 @@ module Gitlab # that should be skipped as they will be removed in a future migration. return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name] - table_schema = Gitlab::Database::GitlabSchema.table_schema(table_name.to_s, undefined: false) - - if table_schema.nil? - error_message = <<~ERROR - No gitlab_schema is defined for the table #{table_name}. Please consider - adding it to the database dictionary. - More info: https://docs.gitlab.com/ee/development/database/database_dictionary.html - ERROR - raise error_message - end + table_schema = Gitlab::Database::GitlabSchema.table_schema!(table_name.to_s) return false unless %i[gitlab_main gitlab_ci].include?(table_schema) diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb new file mode 100644 index 00000000000..63928d7dc09 --- /dev/null +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module ConvertToBigint + # This helper is extracted for the purpose of + # https://gitlab.com/gitlab-org/gitlab/-/issues/392815 + # so that we can test all combinations just once, + # and simplify migration tests. + # + # Once we are done with the PK conversions we can remove this. + def com_or_dev_or_test_but_not_jh? + return true if Gitlab.dev_or_test_env? + + Gitlab.com? && !Gitlab.jh? + end + end + end + end +end diff --git a/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb index 30601bffd7a..2221aea9f46 100644 --- a/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb +++ b/lib/gitlab/database/migration_helpers/loose_foreign_key_helpers.rb @@ -9,11 +9,11 @@ module Gitlab DELETED_RECORDS_INSERT_FUNCTION_NAME = 'insert_into_loose_foreign_keys_deleted_records' def track_record_deletions(table) - execute(<<~SQL) - CREATE TRIGGER #{record_deletion_trigger_name(table)} - AFTER DELETE ON #{table} REFERENCING OLD TABLE AS old_table - FOR EACH STATEMENT - EXECUTE FUNCTION #{DELETED_RECORDS_INSERT_FUNCTION_NAME}(); + execute(<<~SQL.squish) + CREATE TRIGGER #{record_deletion_trigger_name(table)} + AFTER DELETE ON #{table} REFERENCING OLD TABLE AS old_table + FOR EACH STATEMENT + EXECUTE FUNCTION #{DELETED_RECORDS_INSERT_FUNCTION_NAME}(); SQL end @@ -21,6 +21,10 @@ module Gitlab drop_trigger(table, record_deletion_trigger_name(table)) end + def has_loose_foreign_key?(table) + trigger_exists?(table, record_deletion_trigger_name(table)) + end + private def record_deletion_trigger_name(table) diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index b5b8b58681c..07e22963177 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -5,24 +5,6 @@ module Gitlab module MigrationHelpers module V2 include Gitlab::Database::MigrationHelpers - - # Superseded by `create_table` override below - def create_table_with_constraints(*_) - raise <<~EOM - #create_table_with_constraints is not supported anymore - use #create_table instead, for example: - - create_table :db_guides do |t| - t.bigint :stars, default: 0, null: false - t.text :title, limit: 128 - t.text :notes, limit: 1024 - - t.check_constraint 'stars > 1000', name: 'so_many_stars' - end - - See https://docs.gitlab.com/ee/development/database/strings_and_the_text_data_type.html - EOM - end - # Creates a new table, optionally allowing the caller to add text limit constraints to the table. # This method only extends Rails' `create_table` method # @@ -192,6 +174,28 @@ module Gitlab end end + # TRUNCATE is a DDL statement (it drops the table and re-creates it), so we want to run the + # migration in DDL mode, but we also don't want to execute it against all schemas because + # it will be prevented by the lock_writes trigger. + # + # For example, + # a `gitlab_main` table on `:gitlab_main` database will be truncated, + # and a `gitlab_main` table on `:gitlab_ci` database will be skipped. + # + # Note Rails already has a truncate_tables, see + # https://github.com/rails/rails/blob/6-1-stable/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb#L193 + def truncate_tables!(*table_names, connection: self.connection) + table_schemas = Gitlab::Database::GitlabSchema.table_schemas!(table_names) + + raise ArgumentError, "`table_names` must resolve to only one `gitlab_schema`" if table_schemas.size != 1 + + return unless Gitlab::Database.gitlab_schemas_for_connection(connection).include?(table_schemas.first) + + quoted_tables = table_names.map { |table_name| quote_table_name(table_name) }.join(', ') + + execute("TRUNCATE TABLE #{quoted_tables}") + end + private def setup_renamed_column(calling_operation, table, old_column, new_column, type, batch_column_name) diff --git a/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb b/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb new file mode 100644 index 00000000000..2a9d37452bd --- /dev/null +++ b/lib/gitlab/database/migration_helpers/wraparound_vacuum_helpers.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module MigrationHelpers + module WraparoundVacuumHelpers + class WraparoundCheck + WraparoundError = Class.new(StandardError) + + def initialize(table_name, migration:) + @migration = migration + @table_name = table_name + + validate_table_existence! + end + + def execute + return if disabled? + return unless wraparound_vacuum.present? + + log "Autovacuum with wraparound prevention mode is running on `#{table_name}`", title: true + log "This process prevents the migration from acquiring the necessary locks" + log "Query: `#{wraparound_vacuum[:query]}`" + log "Current duration: #{wraparound_vacuum[:duration].inspect}" + log "You can wait until it completes or if absolutely necessary interrupt it, " \ + "but be aware that a new process will kick in immediately, so multiple interruptions " \ + "might be required to time it right with the locks retry mechanism" + end + + private + + attr_reader :table_name + + delegate :say, :connection, to: :@migration + + def wraparound_vacuum + @wraparound_vacuum ||= transform_wraparound_vacuum + end + + def transform_wraparound_vacuum + result = raw_wraparound_vacuum + values = Array.wrap(result.cast_values.first) + + result.columns.zip(values).to_h.with_indifferent_access.compact + end + + def raw_wraparound_vacuum + connection.select_all(<<~SQL.squish) + SELECT age(clock_timestamp(), query_start) as duration, query + FROM postgres_pg_stat_activity_autovacuum() + WHERE query ILIKE '%VACUUM%' || #{quoted_table_name} || '%(to prevent wraparound)' + LIMIT 1 + SQL + end + + def validate_table_existence! + return if connection.table_exists?(table_name) + + raise WraparoundError, "Table #{table_name} does not exist" + end + + def quoted_table_name + connection.quote(table_name) + end + + def disabled? + return true unless wraparound_check_allowed? + + Gitlab::Utils.to_boolean(ENV['GITLAB_MIGRATIONS_DISABLE_WRAPAROUND_CHECK']) + end + + def wraparound_check_allowed? + Gitlab.com? || Gitlab.dev_or_test_env? + end + + def log(text, title: false) + say text, !title + end + end + + def check_if_wraparound_in_progress(table_name) + WraparoundCheck.new(table_name, migration: self).execute + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index e958ce2aba4..cb2a98b553f 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -12,6 +12,7 @@ module Gitlab # For now, these migrations are not considered ready for general use, for more information see the tracking epic: # https://gitlab.com/groups/gitlab-org/-/epics/6751 module BatchedBackgroundMigrationHelpers + NonExistentMigrationError = Class.new(StandardError) BATCH_SIZE = 1_000 # Number of rows to process per job SUB_BATCH_SIZE = 100 # Number of rows to process per sub-batch BATCH_CLASS_NAME = 'PrimaryKeyBatchingStrategy' # Default batch class for batched migrations @@ -200,6 +201,12 @@ module Gitlab def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + if transaction_open? + raise 'The `ensure_batched_background_migration_is_finished` cannot be run inside a transaction. ' \ + 'You can disable transactions by calling `disable_ddl_transaction!` in the body of ' \ + 'your migration class.' + end + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( Gitlab::Database.gitlab_schemas_for_connection(connection), @@ -213,6 +220,10 @@ module Gitlab job_arguments: job_arguments } + if ENV['DBLAB_ENVIRONMENT'] && migration.nil? + raise NonExistentMigrationError, 'called ensure_batched_background_migration_is_finished with non-existent migration name' + end + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? return if migration.finished? diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb index 7b849e3137a..5aafc9f1444 100644 --- a/lib/gitlab/database/migrations/constraints_helpers.rb +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -10,6 +10,27 @@ module Gitlab # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS MAX_IDENTIFIER_NAME_LENGTH = 63 + def self.check_constraint_exists?(table, constraint_name, connection:) + # Constraint names are unique per table in Postgres, not per schema + # Two tables can have constraints with the same name, so we filter by + # the table name in addition to using the constraint_name + + check_sql = <<~SQL + SELECT COUNT(*) + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(connection.current_schema)} + AND rel.relname = #{connection.quote(table)} + SQL + + connection.select_value(check_sql.squish) > 0 + end + # Returns the name for a check constraint # # type: @@ -29,24 +50,7 @@ module Gitlab end def check_constraint_exists?(table, constraint_name) - # Constraint names are unique per table in Postgres, not per schema - # Two tables can have constraints with the same name, so we filter by - # the table name in addition to using the constraint_name - - check_sql = <<~SQL - SELECT COUNT(*) - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - WHERE con.contype = 'c' - AND con.conname = #{connection.quote(constraint_name)} - AND nsp.nspname = #{connection.quote(current_schema)} - AND rel.relname = #{connection.quote(table)} - SQL - - connection.select_value(check_sql) > 0 + ConstraintsHelpers.check_constraint_exists?(table, constraint_name, connection: connection) end # Adds a check constraint to a table diff --git a/lib/gitlab/database/migrations/instrumentation.rb b/lib/gitlab/database/migrations/instrumentation.rb index 8c479d7eda2..5f87bc6bbe2 100644 --- a/lib/gitlab/database/migrations/instrumentation.rb +++ b/lib/gitlab/database/migrations/instrumentation.rb @@ -29,6 +29,8 @@ module Gitlab observation.success = true observation + rescue StandardError => error + observation.error_message = error.message ensure observation.walltime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start diff --git a/lib/gitlab/database/migrations/observation.rb b/lib/gitlab/database/migrations/observation.rb index cd048beac96..1be62496806 100644 --- a/lib/gitlab/database/migrations/observation.rb +++ b/lib/gitlab/database/migrations/observation.rb @@ -4,7 +4,7 @@ module Gitlab module Database module Migrations - Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change, + Observation = Struct.new(:version, :name, :walltime, :success, :total_database_size_change, :error_message, :meta, :query_statistics, keyword_init: true) do def to_json(...) as_json.except('meta').to_json(...) diff --git a/lib/gitlab/database/migrations/pg_backend_pid.rb b/lib/gitlab/database/migrations/pg_backend_pid.rb new file mode 100644 index 00000000000..b59eb55cc6e --- /dev/null +++ b/lib/gitlab/database/migrations/pg_backend_pid.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module PgBackendPid + module MigratorPgBackendPid + extend ::Gitlab::Utils::Override + + override :with_advisory_lock_connection + def with_advisory_lock_connection + super do |conn| + Gitlab::Database::Migrations::PgBackendPid.say(conn) + + yield(conn) + + Gitlab::Database::Migrations::PgBackendPid.say(conn) + end + end + end + + def self.patch! + ActiveRecord::Migrator.prepend(MigratorPgBackendPid) + end + + def self.say(conn) + return unless ActiveRecord::Migration.verbose + + pg_backend_pid = conn.select_value('SELECT pg_backend_pid()') + db_name = Gitlab::Database.db_config_name(conn) + + # rubocop:disable Rails/Output + puts "#{db_name}: == [advisory_lock_connection] " \ + "object_id: #{conn.object_id}, pg_backend_pid: #{pg_backend_pid}" + # rubocop:enable Rails/Output + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb b/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb new file mode 100644 index 00000000000..b5348f4b4e6 --- /dev/null +++ b/lib/gitlab/database/migrations/runner_backoff/active_record_mixin.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module RunnerBackoff + module ActiveRecordMixin + module ActiveRecordMigrationProxyRunnerBackoff + # Regular AR migrations don't have this, + # only ones inheriting from Gitlab::Database::Migration have + def enable_runner_backoff? + !!migration.try(:enable_runner_backoff?) + end + end + + module ActiveRecordMigratorRunnerBackoff + def execute_migration_in_transaction(migration) + if migration.enable_runner_backoff? + RunnerBackoff::Communicator.execute_with_lock(migration) { super } + else + super + end + end + end + + def self.patch! + ActiveRecord::MigrationProxy.prepend(ActiveRecordMigrationProxyRunnerBackoff) + ActiveRecord::Migrator.prepend(ActiveRecordMigratorRunnerBackoff) + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner_backoff/communicator.rb b/lib/gitlab/database/migrations/runner_backoff/communicator.rb new file mode 100644 index 00000000000..874dfc56832 --- /dev/null +++ b/lib/gitlab/database/migrations/runner_backoff/communicator.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module RunnerBackoff + class Communicator + EXPIRY = 1.minute + KEY = 'gitlab/database/migration/runner/backoff' + + def self.execute_with_lock(migration, &block) + new(migration).execute_with_lock(&block) + end + + def self.backoff_runner? + return false if ::Feature.disabled?(:runner_migrations_backoff, type: :ops) + + Gitlab::ExclusiveLease.new(KEY, timeout: EXPIRY).exists? + end + + def initialize(migration, logger: Gitlab::AppLogger) + @migration = migration + @logger = logger + end + + def execute_with_lock + log(message: 'Executing migration with Runner backoff') + + set_lock + yield if block_given? + ensure + remove_lock + end + + private + + attr_reader :logger, :migration + + def set_lock + raise 'Could not set backoff key' unless exclusive_lease.try_obtain + + log(message: 'Runner backoff key is set') + end + + def remove_lock + exclusive_lease.cancel + + log(message: 'Runner backoff key was removed') + end + + def exclusive_lease + @exclusive_lease ||= Gitlab::ExclusiveLease.new(KEY, timeout: EXPIRY) + end + + def log(params) + logger.info(log_params.merge(params)) + end + + def log_params + { class: migration.name } + end + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb b/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb new file mode 100644 index 00000000000..b9e564d85e6 --- /dev/null +++ b/lib/gitlab/database/migrations/runner_backoff/migration_helpers.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module RunnerBackoff + module MigrationHelpers + extend ActiveSupport::Concern + + class_methods do + def enable_runner_backoff! + @enable_runner_backoff = true # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def enable_runner_backoff? + !!@enable_runner_backoff + end + end + + delegate :enable_runner_backoff?, to: :class + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/sidekiq_helpers.rb b/lib/gitlab/database/migrations/sidekiq_helpers.rb index c536b33bbdf..6ab14c0fbe6 100644 --- a/lib/gitlab/database/migrations/sidekiq_helpers.rb +++ b/lib/gitlab/database/migrations/sidekiq_helpers.rb @@ -25,9 +25,14 @@ module Gitlab times_in_a_row: DEFAULT_TIMES_IN_A_ROW, max_attempts: DEFAULT_MAX_ATTEMPTS ) - kwargs = { times_in_a_row: times_in_a_row, max_attempts: max_attempts } + if transaction_open? + raise 'sidekiq_remove_jobs can not be run inside a transaction, ' \ + 'you can disable transactions by calling disable_ddl_transaction! ' \ + 'in the body of your migration class' + end + job_klasses_queues = job_klasses .select { |job_klass| job_klass.to_s.safe_constantize.present? } .map { |job_klass| job_klass.safe_constantize.queue } diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index 01fdba22c19..af853c933ba 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -27,7 +27,7 @@ module Gitlab table_max_value = define_batchable_model(migration.table_name, connection: connection) .maximum(migration.column_name) - largest_batch_start = table_max_value - migration.batch_size + largest_batch_start = [table_max_value - migration.batch_size, smallest_batch_start].max # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 # to pick actual batches to sample. diff --git a/lib/gitlab/database/obsolete_ignored_columns.rb b/lib/gitlab/database/obsolete_ignored_columns.rb index 2b88ab12380..c172d15301a 100644 --- a/lib/gitlab/database/obsolete_ignored_columns.rb +++ b/lib/gitlab/database/obsolete_ignored_columns.rb @@ -2,15 +2,15 @@ module Gitlab module Database - # Checks which `ignored_columns` can be safely removed by scanning - # the current schema for all `ApplicationRecord` descendants. + # Checks which `ignored_columns` definitions can be safely removed by + # scanning the current schema for all `ApplicationRecord` descendants. class ObsoleteIgnoredColumns def initialize(base = ApplicationRecord) @base = base end def execute - @base.descendants.map do |klass| + @base.descendants.filter_map do |klass| next if klass.abstract_class? safe_to_remove = ignored_columns_safe_to_remove_for(klass) diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 6314aff9914..7222f148b3f 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -27,6 +27,8 @@ module Gitlab end def sync_partitions(models_to_sync = registered_for_sync, only_on: nil) + return unless Feature.enabled?(:partition_manager_sync_partitions, type: :ops) + Gitlab::AppLogger.info(message: 'Syncing dynamic postgres partitions') Gitlab::Database::EachDatabase.each_model_connection(models_to_sync, only_on: only_on) do |model| @@ -37,8 +39,9 @@ module Gitlab models_to_sync.each do |model| next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) + model_connection_name = model.connection_db_config.name Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| - if connection_name != model.connection_db_config.name + if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end end diff --git a/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb new file mode 100644 index 00000000000..69a69091b5c --- /dev/null +++ b/lib/gitlab/database/partitioning/ci_sliding_list_strategy.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class CiSlidingListStrategy < SlidingListStrategy + def initial_partition + partition_for(100) + end + + def next_partition + partition_for(active_partition.value + 1) + end + + def validate_and_fix; end + + def after_adding_partitions; end + + def extra_partitions + [] + end + + private + + def ensure_partitioning_column_ignored_or_readonly!; end + + def partition_for(value) + SingleNumericListPartition.new(table_name, value, partition_name: partition_name(value)) + end + + def partition_name(value) + [ + table_name.to_s.delete_prefix('p_'), + value + ].join('_') + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb deleted file mode 100644 index 58447481e60..00000000000 --- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb +++ /dev/null @@ -1,268 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module Partitioning - class ConvertTableToFirstListPartition - UnableToPartition = Class.new(StandardError) - - SQL_STATEMENT_SEPARATOR = ";\n\n" - - attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value - - def initialize( - migration_context:, table_name:, parent_table_name:, partitioning_column:, - zero_partition_value:, lock_tables: []) - - @migration_context = migration_context - @connection = migration_context.connection - @table_name = table_name - @parent_table_name = parent_table_name - @partitioning_column = partitioning_column - @zero_partition_value = zero_partition_value - @lock_tables = Array.wrap(lock_tables) - end - - def prepare_for_partitioning - assert_existing_constraints_partitionable - - add_partitioning_check_constraint - end - - def revert_preparation_for_partitioning - migration_context.remove_check_constraint(table_name, partitioning_constraint.name) - end - - def partition - assert_existing_constraints_partitionable - assert_partitioning_constraint_present - create_parent_table - attach_foreign_keys_to_parent - - lock_args = { - raise_on_exhaustion: true, - timing_configuration: lock_timing_configuration - } - - migration_context.with_lock_retries(**lock_args) do - migration_context.execute(sql_to_convert_table) - end - end - - def revert_partitioning - migration_context.with_lock_retries(raise_on_exhaustion: true) do - migration_context.execute(<<~SQL) - ALTER TABLE #{connection.quote_table_name(parent_table_name)} - DETACH PARTITION #{connection.quote_table_name(table_name)}; - SQL - - alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name) - .join(SQL_STATEMENT_SEPARATOR) - - migration_context.execute(alter_sequences_sql) - - # This takes locks for all the foreign keys that the parent table had. - # However, those same locks were taken while detaching the partition, and we can't avoid that. - # If we dropped the foreign key before detaching the partition to avoid this locking, - # the drop would cascade to the child partitions and drop their foreign keys as well - migration_context.drop_table(parent_table_name) - end - - add_partitioning_check_constraint - end - - private - - attr_reader :connection, :migration_context - - delegate :quote_table_name, :quote_column_name, to: :connection - - def sql_to_convert_table - # The critical statement here is the attach_table_to_parent statement. - # The following statements could be run in a later transaction, - # but they acquire the same locks so it's much faster to incude them - # here. - [ - lock_tables_statement, - attach_table_to_parent_statement, - alter_sequence_statements(old_table: table_name, new_table: parent_table_name), - remove_constraint_statement - ].flatten.join(SQL_STATEMENT_SEPARATOR) - end - - def table_identifier - "#{connection.current_schema}.#{table_name}" - end - - def assert_existing_constraints_partitionable - violating_constraints = Gitlab::Database::PostgresConstraint - .by_table_identifier(table_identifier) - .primary_or_unique_constraints - .not_including_column(partitioning_column) - .to_a - - return if violating_constraints.empty? - - violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" } - - raise UnableToPartition, <<~MSG - Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column} - - All primary key and unique constraints must include the partitioning column. - Violations: - #{violation_messages.join("\n")} - MSG - end - - def partitioning_constraint - constraints_on_column = Gitlab::Database::PostgresConstraint - .by_table_identifier(table_identifier) - .check_constraints - .valid - .including_column(partitioning_column) - - constraints_on_column.to_a.find do |constraint| - constraint.definition == "CHECK ((#{partitioning_column} = #{zero_partition_value}))" - end - end - - def assert_partitioning_constraint_present - return if partitioning_constraint - - raise UnableToPartition, <<~MSG - Table #{table_name} is not ready for partitioning. - Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value}) - MSG - end - - def add_partitioning_check_constraint - return if partitioning_constraint.present? - - check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}" - # Any constraint name would work. The constraint is found based on its definition before partitioning - migration_context.add_check_constraint(table_name, check_body, 'partitioning_constraint') - - raise UnableToPartition, 'Error adding partitioning constraint' unless partitioning_constraint.present? - end - - def create_parent_table - migration_context.execute(<<~SQL) - CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} ( - LIKE #{quote_table_name(table_name)} INCLUDING ALL - ) PARTITION BY LIST(#{quote_column_name(partitioning_column)}) - SQL - end - - def attach_foreign_keys_to_parent - migration_context.foreign_keys(table_name).each do |fk| - # At this point no other connection knows about the parent table. - # Thus the only contended lock in the following transaction is on fk.to_table. - # So a deadlock is impossible. - - # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist. - # Don't try to recreate it in that case - if migration_context.foreign_keys(parent_table_name) - .any? { |p_fk| p_fk.options[:name] == fk.options[:name] } - next - end - - migration_context.with_lock_retries(raise_on_exhaustion: true) do - migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options) - end - end - end - - def lock_tables_statement - return if @lock_tables.empty? - - table_names = @lock_tables.map { |name| quote_table_name(name) }.join(', ') - - <<~SQL - LOCK #{table_names} IN ACCESS EXCLUSIVE MODE - SQL - end - - def attach_table_to_parent_statement - <<~SQL - ALTER TABLE #{quote_table_name(parent_table_name)} - ATTACH PARTITION #{table_name} - FOR VALUES IN (#{zero_partition_value}) - SQL - end - - def alter_sequence_statements(old_table:, new_table:) - sequences_owned_by(old_table).map do |seq_info| - seq_name, column_name = seq_info.values_at(:name, :column_name) - - statement_parts = [] - - # If a different user owns the old table, the conversion process will fail to reassign the sequence - # ownership to the new parent table (as it will be owned by the current user). - # Force the old table to be owned by the current user in that case. - unless current_user_owns_table?(old_table) - statement_parts << set_current_user_owns_table_statement(old_table) - end - - statement_parts << <<~SQL.chomp - ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} - SQL - - statement_parts.join(SQL_STATEMENT_SEPARATOR) - end - end - - def remove_constraint_statement - <<~SQL - ALTER TABLE #{quote_table_name(parent_table_name)} - DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)} - SQL - end - - # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887 - def sequences_owned_by(table_name) - sequence_data = connection.exec_query(<<~SQL, nil, [table_name]) - SELECT seq_pg_class.relname AS seq_name, - dep_pg_class.relname AS table_name, - pg_attribute.attname AS col_name - FROM pg_class seq_pg_class - INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid - INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid - INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid - AND pg_depend.refobjsubid = pg_attribute.attnum - WHERE seq_pg_class.relkind = 'S' - AND dep_pg_class.relname = $1 - SQL - - sequence_data.map do |seq_info| - name, column_name = seq_info.values_at('seq_name', 'col_name') - { name: name, column_name: column_name } - end - end - - def table_owner(table_name) - connection.select_value(<<~SQL, nil, [table_name]) - SELECT tableowner FROM pg_tables WHERE tablename = $1 - SQL - end - - def current_user_owns_table?(table_name) - current_user = connection.select_value('select current_user') - table_owner(table_name) == current_user - end - - def set_current_user_owns_table_statement(table_name) - <<~SQL.chomp - ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER - SQL - end - - def lock_timing_configuration - iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION - aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] } - - iterations + aggressive_iterations - end - end - end - end -end diff --git a/lib/gitlab/database/partitioning/list/convert_table.rb b/lib/gitlab/database/partitioning/list/convert_table.rb new file mode 100644 index 00000000000..d4fb150d956 --- /dev/null +++ b/lib/gitlab/database/partitioning/list/convert_table.rb @@ -0,0 +1,317 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + module List + class ConvertTable + UnableToPartition = Class.new(StandardError) + + SQL_STATEMENT_SEPARATOR = ";\n\n" + + PARTITIONING_CONSTRAINT_NAME = 'partitioning_constraint' + + attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value, + :locking_configuration + + def initialize( + migration_context:, table_name:, parent_table_name:, partitioning_column:, + zero_partition_value:, lock_tables: []) + + @migration_context = migration_context + @connection = migration_context.connection + @table_name = table_name + @parent_table_name = parent_table_name + @partitioning_column = partitioning_column + @zero_partition_value = zero_partition_value + @locking_configuration = LockingConfiguration.new(migration_context, table_locking_order: lock_tables) + end + + def prepare_for_partitioning(async: false) + assert_existing_constraints_partitionable + + add_partitioning_check_constraint(async: async) + end + + def revert_preparation_for_partitioning + migration_context.remove_check_constraint(table_name, partitioning_constraint.name) + end + + def partition + assert_existing_constraints_partitionable + assert_partitioning_constraint_present + + create_parent_table + attach_foreign_keys_to_parent + locking_sql = locking_configuration.locking_statement_for(tables_that_will_lock_during_partitioning) + + locking_configuration.with_lock_retries do + # Loose FKs trigger will exclusively lock the table and it might + # not follow the locking order needed to attach the partition. + migration_context.execute(locking_sql) if locking_sql.present? + + redefine_loose_foreign_key_triggers do + migration_context.execute(sql_to_convert_table) + end + end + end + + def revert_partitioning + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.execute(<<~SQL) + ALTER TABLE #{connection.quote_table_name(parent_table_name)} + DETACH PARTITION #{connection.quote_table_name(table_name)}; + SQL + + alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name) + .join(SQL_STATEMENT_SEPARATOR) + + migration_context.execute(alter_sequences_sql) + + # This takes locks for all the foreign keys that the parent table had. + # However, those same locks were taken while detaching the partition, and we can't avoid that. + # If we dropped the foreign key before detaching the partition to avoid this locking, + # the drop would cascade to the child partitions and drop their foreign keys as well + migration_context.drop_table(parent_table_name) + end + + add_partitioning_check_constraint + end + + private + + attr_reader :connection, :migration_context + + delegate :quote_table_name, :quote_column_name, :current_schema, to: :connection + + def sql_to_convert_table + # The critical statement here is the attach_table_to_parent statement. + # The following statements could be run in a later transaction, + # but they acquire the same locks so it's much faster to include them + # here. + [ + attach_table_to_parent_statement, + alter_sequence_statements(old_table: table_name, new_table: parent_table_name), + remove_constraint_statement + ].flatten.join(SQL_STATEMENT_SEPARATOR) + end + + def table_identifier + "#{current_schema}.#{table_name}" + end + + def assert_existing_constraints_partitionable + violating_constraints = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .primary_or_unique_constraints + .not_including_column(partitioning_column) + .to_a + + return if violating_constraints.empty? + + violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" } + + raise UnableToPartition, <<~MSG + Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column} + + All primary key and unique constraints must include the partitioning column. + Violations: + #{violation_messages.join("\n")} + MSG + end + + def partitioning_constraint + constraints_on_column = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .check_constraints + .including_column(partitioning_column) + + check_body = "CHECK ((#{partitioning_column} = #{zero_partition_value}))" + + constraints_on_column.find do |constraint| + constraint.definition.start_with?(check_body) + end + end + + def assert_partitioning_constraint_present + return if partitioning_constraint&.constraint_valid? + + raise UnableToPartition, <<~MSG + Table #{table_name} is not ready for partitioning. + Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value}) + MSG + end + + def add_partitioning_check_constraint(async: false) + return validate_partitioning_constraint_synchronously if partitioning_constraint.present? + + check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}" + # Any constraint name would work. The constraint is found based on its definition before partitioning + migration_context.add_check_constraint( + table_name, check_body, PARTITIONING_CONSTRAINT_NAME, + validate: !async + ) + + if async + migration_context.prepare_async_check_constraint_validation( + table_name, name: PARTITIONING_CONSTRAINT_NAME + ) + end + + return if partitioning_constraint.present? + + raise UnableToPartition, <<~MSG + Error adding partitioning constraint `#{PARTITIONING_CONSTRAINT_NAME}` for `#{table_name}` + MSG + end + + def validate_partitioning_constraint_synchronously + if partitioning_constraint.constraint_valid? + return Gitlab::AppLogger.info <<~MSG + Nothing to do, the partitioning constraint exists and is valid for `#{table_name}` + MSG + end + + # Async validations are executed only on .com, we need to validate synchronously for self-managed + migration_context.validate_check_constraint(table_name, partitioning_constraint.name) + return if partitioning_constraint.constraint_valid? + + raise UnableToPartition, <<~MSG + Error validating partitioning constraint `#{partitioning_constraint.name}` for `#{table_name}` + MSG + end + + def create_parent_table + migration_context.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} ( + LIKE #{quote_table_name(table_name)} INCLUDING ALL + ) PARTITION BY LIST(#{quote_column_name(partitioning_column)}) + SQL + end + + def attach_foreign_keys_to_parent + migration_context.foreign_keys(table_name).each do |fk| + # At this point no other connection knows about the parent table. + # Thus the only contended lock in the following transaction is on fk.to_table. + # So a deadlock is impossible. + + # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist + # Don't try to recreate it in that case + if migration_context.foreign_keys(parent_table_name) + .any? { |p_fk| p_fk.options[:name] == fk.options[:name] } + next + end + + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options) + end + end + end + + def attach_table_to_parent_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + ATTACH PARTITION #{table_name} + FOR VALUES IN (#{zero_partition_value}) + SQL + end + + def alter_sequence_statements(old_table:, new_table:) + sequences_owned_by(old_table).map do |seq_info| + seq_name, column_name = seq_info.values_at(:name, :column_name) + + statement_parts = [] + + # If a different user owns the old table, the conversion process will fail to reassign the sequence + # ownership to the new parent table (as it will be owned by the current user). + # Force the old table to be owned by the current user in that case. + unless current_user_owns_table?(old_table) + statement_parts << set_current_user_owns_table_statement(old_table) + end + + statement_parts << <<~SQL.chomp + ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} + SQL + + statement_parts.join(SQL_STATEMENT_SEPARATOR) + end + end + + def remove_constraint_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)} + SQL + end + + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887 + def sequences_owned_by(table_name) + sequence_data = connection.exec_query(<<~SQL, nil, [table_name]) + SELECT seq_pg_class.relname AS seq_name, + dep_pg_class.relname AS table_name, + pg_attribute.attname AS col_name + FROM pg_class seq_pg_class + INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid + INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid + INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid + AND pg_depend.refobjsubid = pg_attribute.attnum + WHERE seq_pg_class.relkind = 'S' + AND dep_pg_class.relname = $1 + SQL + + sequence_data.map do |seq_info| + name, column_name = seq_info.values_at('seq_name', 'col_name') + { name: name, column_name: column_name } + end + end + + def table_owner(table_name) + connection.select_value(<<~SQL, nil, [table_name]) + SELECT tableowner FROM pg_tables WHERE tablename = $1 + SQL + end + + def current_user_owns_table?(table_name) + current_user = connection.select_value('select current_user') + table_owner(table_name) == current_user + end + + def set_current_user_owns_table_statement(table_name) + <<~SQL.chomp + ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER + SQL + end + + def table_name_for_identifier(table_identifier) + /^\w+\.(\w+)*$/.match(table_identifier)[1] + end + + def redefine_loose_foreign_key_triggers + if migration_context.has_loose_foreign_key?(table_name) + migration_context.untrack_record_deletions(table_name) + + yield if block_given? + + migration_context.track_record_deletions(parent_table_name) + migration_context.track_record_deletions(table_name) + elsif block_given? + yield + end + end + + def tables_that_will_lock_during_partitioning + # Locks are taken against the table + all tables that reference it by foreign key + # postgres_foreign_keys.referenced_table_name gives the table name that we need here directly, but that + # column did not exist yet during the migration 20221021145820_create_routing_table_for_builds_metadata_v2 + # To ensure compatibility with that migration if it is run with this code, use referenced_table_identifier + # here. + referenced_tables = Gitlab::Database::PostgresForeignKey + .by_constrained_table_identifier(table_identifier) + .map { |fk| table_name_for_identifier(fk.referenced_table_identifier) } + referenced_tables + [table_name] + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/list/locking_configuration.rb b/lib/gitlab/database/partitioning/list/locking_configuration.rb new file mode 100644 index 00000000000..02d20383de4 --- /dev/null +++ b/lib/gitlab/database/partitioning/list/locking_configuration.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + module List + class LockingConfiguration + attr_reader :migration_context + + def initialize(migration_context, table_locking_order:) + @migration_context = migration_context + @table_locking_order = table_locking_order.map(&:to_s) + assert_table_names_unqualified!(@table_locking_order) + end + + def locking_statement_for(tables) + tables_to_lock = locking_order_for(tables) + + return if tables_to_lock.empty? + + table_names = tables_to_lock.map { |name| migration_context.quote_table_name(name) }.join(', ') + + <<~SQL + LOCK #{table_names} IN ACCESS EXCLUSIVE MODE + SQL + end + + # Sorts and subsets `tables` to the tables that were explicitly requested for locking + # in the order that that locking was requested. + def locking_order_for(tables) + tables = Array.wrap(tables) + assert_table_names_unqualified!(tables) + + @table_locking_order.intersection(tables.map(&:to_s)) + end + + def lock_timing_configuration + iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION + aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] } + + iterations + aggressive_iterations + end + + def with_lock_retries(&block) + lock_args = { + raise_on_exhaustion: true, + timing_configuration: lock_timing_configuration + } + + migration_context.with_lock_retries(**lock_args, &block) + end + + private + + def assert_table_names_unqualified!(table_names) + tables = Array.wrap(table_names).select { |name| name.to_s.include?('.') } + return if tables.empty? + + raise ArgumentError, "All table names must be unqualified, but #{tables.join(', ')} include schema" + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index 55ca9ff8645..124fae582d3 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -34,6 +34,8 @@ module Gitlab create(partitions_to_create) unless partitions_to_create.empty? detach(partitions_to_detach) unless partitions_to_detach.empty? end + rescue ArgumentError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) rescue StandardError => e Gitlab::AppLogger.error( message: "Failed to create / detach partition(s)", diff --git a/lib/gitlab/database/partitioning/replace_table.rb b/lib/gitlab/database/partitioning/replace_table.rb index 21a175a660d..60871a64845 100644 --- a/lib/gitlab/database/partitioning/replace_table.rb +++ b/lib/gitlab/database/partitioning/replace_table.rb @@ -70,7 +70,7 @@ module Gitlab def rename_partitions(statements, old_table_name, new_table_name) Gitlab::Database::PostgresPartition.for_parent_table(old_table_name).each do |partition| - new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name) + new_partition_name = partition.name.sub(/#{old_table_name}/, new_table_name.to_s) old_primary_key = default_primary_key(partition.name) new_primary_key = default_primary_key(new_partition_name) diff --git a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb index dcf457b9d63..e87707953ae 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/backfill_partitioned_table.rb @@ -21,7 +21,7 @@ module Gitlab return end - bulk_copy = BulkCopy.new(source_table, partitioned_table, source_column, connection: connection) + bulk_copy = Gitlab::Database::PartitioningMigrationHelpers::BulkCopy.new(source_table, partitioned_table, source_column, connection: connection) parent_batch_relation = relation_scoped_to_range(source_table, source_column, start_id, stop_id) parent_batch_relation.each_batch(of: SUB_BATCH_SIZE) do |sub_batch| @@ -56,41 +56,6 @@ module Gitlab def mark_jobs_as_succeeded(*arguments) BackgroundMigrationJob.mark_all_as_succeeded(self.class.name, arguments) end - - # Helper class to copy data between two tables via upserts - class BulkCopy - DELIMITER = ', ' - - attr_reader :source_table, :destination_table, :source_column, :connection - - def initialize(source_table, destination_table, source_column, connection:) - @source_table = source_table - @destination_table = destination_table - @source_column = source_column - @connection = connection - end - - def copy_between(start_id, stop_id) - connection.execute(<<~SQL) - INSERT INTO #{destination_table} (#{column_listing}) - SELECT #{column_listing} - FROM #{source_table} - WHERE #{source_column} BETWEEN #{start_id} AND #{stop_id} - FOR UPDATE - ON CONFLICT (#{conflict_targets}) DO NOTHING - SQL - end - - private - - def column_listing - @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER) - end - - def conflict_targets - connection.primary_key(destination_table).join(DELIMITER) - end - end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb b/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb new file mode 100644 index 00000000000..b8f5a2e3ad4 --- /dev/null +++ b/lib/gitlab/database/partitioning_migration_helpers/bulk_copy.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module PartitioningMigrationHelpers + # Helper class to copy data between two tables via upserts + class BulkCopy + DELIMITER = ', ' + + attr_reader :source_table, :destination_table, :source_column, :connection + + def initialize(source_table, destination_table, source_column, connection:) + @source_table = source_table + @destination_table = destination_table + @source_column = source_column + @connection = connection + end + + def copy_between(start_id, stop_id) + connection.execute(<<~SQL) + INSERT INTO #{destination_table} (#{column_listing}) + SELECT #{column_listing} + FROM #{source_table} + WHERE #{source_column} BETWEEN #{start_id} AND #{stop_id} + FOR UPDATE + ON CONFLICT (#{conflict_targets}) DO NOTHING + SQL + end + + private + + def column_listing + @column_listing ||= connection.columns(source_table).map(&:name).join(DELIMITER) + end + + def conflict_targets + connection.primary_keys(destination_table).join(DELIMITER) + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb index 8849191f356..7d9c12d776e 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/foreign_key_helpers.rb @@ -32,46 +32,75 @@ module Gitlab # column - The name of the column to create the foreign key on. # on_delete - The action to perform when associated data is removed, # defaults to "CASCADE". + # on_update - The action to perform when associated data is updated, + # no default value is set. # name - The name of the foreign key. + # validate - Flag that controls whether the new foreign key will be + # validated after creation and if it will be added on the parent table. + # If the flag is not set, the constraint will only be enforced for new data + # in the existing partitions. The helper will need to be called again + # with the flag set to `true` to add the foreign key on the parent table + # after validating it on all partitions. + # `validate: false` should be paired with `prepare_partitioned_async_foreign_key_validation` + # reverse_lock_order - Flag that controls whether we should attempt to acquire locks in the reverse + # order of the ALTER TABLE. This can be useful in situations where the foreign + # key creation could deadlock with another process. # - def add_concurrent_partitioned_foreign_key(source, target, column:, on_delete: :cascade, name: nil) + def add_concurrent_partitioned_foreign_key(source, target, column:, **options) assert_not_in_transaction_block(scope: ERROR_SCOPE) - partition_options = { - column: column, - on_delete: on_delete, + options.reverse_merge!({ + target_column: :id, + on_delete: :cascade, + on_update: nil, + name: nil, + validate: true, + reverse_lock_order: false, + column: column + }) - # We'll use the same FK name for all partitions and match it to - # the name used for the partitioned table to follow the convention - # used by PostgreSQL when adding FKs to new partitions - name: name.presence || concurrent_partitioned_foreign_key_name(source, column), + # We'll use the same FK name for all partitions and match it to + # the name used for the partitioned table to follow the convention + # used by PostgreSQL when adding FKs to new partitions + options[:name] ||= concurrent_partitioned_foreign_key_name(source, column) + check_options = options.slice(:column, :on_delete, :on_update, :name) + check_options[:primary_key] = options[:target_column] - # Force the FK validation to true for partitions (and the partitioned table) - validate: true - } - - if foreign_key_exists?(source, target, **partition_options) + if foreign_key_exists?(source, target, **check_options) warning_message = "Foreign key not created because it exists already " \ "(this may be due to an aborted migration or similar): " \ - "source: #{source}, target: #{target}, column: #{partition_options[:column]}, "\ - "name: #{partition_options[:name]}, on_delete: #{partition_options[:on_delete]}" + "source: #{source}, target: #{target}, column: #{options[:column]}, "\ + "name: #{options[:name]}, on_delete: #{options[:on_delete]}, "\ + "on_update: #{options[:on_update]}" Gitlab::AppLogger.warn warning_message return end - partitioned_table = find_partitioned_table(source) - - partitioned_table.postgres_partitions.order(:name).each do |partition| - add_concurrent_foreign_key(partition.identifier, target, **partition_options) + Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition| + add_concurrent_foreign_key(partition.identifier, target, **options) end - with_lock_retries do - add_foreign_key(source, target, **partition_options) + # If we are to add the FK on the parent table now, it will trigger + # the validation on all partitions. The helper must be called one + # more time with `validate: true` after the FK is valid on all partitions. + return unless options[:validate] + + options[:allow_partitioned] = true + add_concurrent_foreign_key(source, target, **options) + end + + def validate_partitioned_foreign_key(source, column, name: nil) + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + Gitlab::Database::PostgresPartitionedTable.each_partition(source) do |partition| + validate_foreign_key(partition.identifier, column, name: name) end end + private + # Returns the name for a concurrent partitioned foreign key. # # Similar to concurrent_foreign_key_name (Gitlab::Database::MigrationHelpers) diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index f9790bf53b9..61e95dbe1a4 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -6,19 +6,16 @@ module Gitlab module TableManagementHelpers include ::Gitlab::Database::SchemaHelpers include ::Gitlab::Database::MigrationHelpers + include ::Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze ERROR_SCOPE = 'table partitioning' MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable" + MIGRATION = "BackfillPartitionedTable" BATCH_INTERVAL = 2.minutes.freeze BATCH_SIZE = 50_000 - - JobArguments = Struct.new(:start_id, :stop_id, :source_table_name, :partitioned_table_name, :source_column) do - def self.from_array(arguments) - self.new(*arguments) - end - end + SUB_BATCH_SIZE = 2_500 # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on @@ -107,12 +104,21 @@ module Gitlab partitioned_table_name = make_partitioned_table_name(table_name) primary_key = connection.primary_key(table_name) - enqueue_background_migration(table_name, partitioned_table_name, primary_key) + + queue_batched_background_migration( + MIGRATION, + table_name, + primary_key, + partitioned_table_name, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE, + job_interval: BATCH_INTERVAL + ) end # Cleanup a previously enqueued background migration to copy data into a partitioned table. This will not # prevent the enqueued jobs from executing, but instead cleans up information in the database used to track the - # state of the background migration. It should be safe to also remove the partitioned table even if the + # state of the batched background migration. It should be safe to also remove the partitioned table even if the # background jobs are still in-progress, as the absence of the table will cause them to safely exit. # # Example: @@ -122,7 +128,10 @@ module Gitlab def cleanup_partitioning_data_migration(table_name) assert_table_is_allowed(table_name) - cleanup_migration_jobs(table_name) + partitioned_table_name = make_partitioned_table_name(table_name) + primary_key = connection.primary_key(table_name) + + delete_batched_background_migration(MIGRATION, table_name, primary_key, [partitioned_table_name]) end def create_hash_partitions(table_name, number_of_partitions) @@ -142,14 +151,11 @@ module Gitlab end end - # Executes cleanup tasks from a previous BackgroundMigration to backfill a partitioned table by finishing - # pending jobs and performing a final data synchronization. - # This performs two steps: - # 1. Wait to finish any pending BackgroundMigration jobs that have not succeeded - # 2. Inline copy any missed rows from the original table to the partitioned table + # Executes jobs from previous BatchedBackgroundMigration to backfill the partitioned table by finishing + # pending jobs. # # **NOTE** Migrations using this method cannot be scheduled in the same release as the migration that - # schedules the background migration using the `enqueue_background_migration` helper, or else the + # schedules the background migration using the `enqueue_partitioning_data_migration` helper, or else the # background migration jobs will be force-executed. # # Example: @@ -157,23 +163,21 @@ module Gitlab # finalize_backfilling_partitioned_table :audit_events # def finalize_backfilling_partitioned_table(table_name) - Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - assert_table_is_allowed(table_name) - assert_not_in_transaction_block(scope: ERROR_SCOPE) partitioned_table_name = make_partitioned_table_name(table_name) + unless table_exists?(partitioned_table_name) raise "could not find partitioned table for #{table_name}, " \ "this could indicate the previous partitioning migration has been rolled back." end - Gitlab::BackgroundMigration.steal(MIGRATION_CLASS_NAME) do |background_job| - JobArguments.from_array(background_job.args.second).source_table_name == table_name.to_s - end - - primary_key = connection.primary_key(table_name) - copy_missed_records(table_name, partitioned_table_name, primary_key) + ensure_batched_background_migration_is_finished( + job_class_name: MIGRATION, + table_name: table_name, + column_name: connection.primary_key(table_name), + job_arguments: [partitioned_table_name] + ) Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do disable_statement_timeout do @@ -251,22 +255,22 @@ module Gitlab create_sync_trigger(source_table_name, trigger_name, function_name) end - def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, async: false) validate_not_in_transaction!(:prepare_constraint_for_list_partitioning) - Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + Gitlab::Database::Partitioning::List::ConvertTable .new(migration_context: self, table_name: table_name, parent_table_name: parent_table_name, partitioning_column: partitioning_column, zero_partition_value: initial_partitioning_value - ).prepare_for_partitioning + ).prepare_for_partitioning(async: async) end def revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) validate_not_in_transaction!(:revert_preparing_constraint_for_list_partitioning) - Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + Gitlab::Database::Partitioning::List::ConvertTable .new(migration_context: self, table_name: table_name, parent_table_name: parent_table_name, @@ -278,7 +282,7 @@ module Gitlab def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: []) validate_not_in_transaction!(:convert_table_to_first_list_partition) - Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + Gitlab::Database::Partitioning::List::ConvertTable .new(migration_context: self, table_name: table_name, parent_table_name: parent_table_name, @@ -291,7 +295,7 @@ module Gitlab def revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) validate_not_in_transaction!(:revert_converting_table_to_first_list_partition) - Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + Gitlab::Database::Partitioning::List::ConvertTable .new(migration_context: self, table_name: table_name, parent_table_name: parent_table_name, @@ -444,45 +448,6 @@ module Gitlab create_trigger(table_name, trigger_name, function_name, fires: 'AFTER INSERT OR UPDATE OR DELETE') end - def enqueue_background_migration(source_table_name, partitioned_table_name, source_column) - source_model = define_batchable_model(source_table_name) - - queue_background_migration_jobs_by_range_at_intervals( - source_model, - MIGRATION_CLASS_NAME, - BATCH_INTERVAL, - batch_size: BATCH_SIZE, - other_job_arguments: [source_table_name.to_s, partitioned_table_name, source_column], - track_jobs: true) - end - - def cleanup_migration_jobs(table_name) - ::Gitlab::Database::BackgroundMigrationJob.for_partitioning_migration(MIGRATION_CLASS_NAME, table_name).delete_all - end - - def copy_missed_records(source_table_name, partitioned_table_name, source_column) - backfill_table = BackfillPartitionedTable.new(connection: connection) - - relation = ::Gitlab::Database::BackgroundMigrationJob.pending - .for_partitioning_migration(MIGRATION_CLASS_NAME, source_table_name) - - relation.each_batch do |batch| - batch.each do |pending_migration_job| - job_arguments = JobArguments.from_array(pending_migration_job.arguments) - start_id = job_arguments.start_id - stop_id = job_arguments.stop_id - - say("Backfilling data into partitioned table for ids from #{start_id} to #{stop_id}") - job_updated_count = backfill_table.perform(start_id, stop_id, source_table_name, - partitioned_table_name, source_column) - - unless job_updated_count > 0 - raise "failed to update tracking record for ids from #{start_id} to #{stop_id}" - end - end - end - end - def replace_table(original_table_name, replacement_table_name, replaced_table_name, primary_key_name) replace_table = Gitlab::Database::Partitioning::ReplaceTable.new(connection, original_table_name.to_s, replacement_table_name, replaced_table_name, primary_key_name) diff --git a/lib/gitlab/database/pg_depend.rb b/lib/gitlab/database/pg_depend.rb new file mode 100644 index 00000000000..5f3cdaeb979 --- /dev/null +++ b/lib/gitlab/database/pg_depend.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class PgDepend < SharedModel + self.table_name = 'pg_depend' + + TYPES = { + 'VIEW' => %w[v m].freeze + }.freeze + + scope :from_pg_extension, ->(type = nil) do + joins('INNER JOIN pg_class ON pg_class.oid = pg_depend.objid') + .where(pg_class: { relkind: TYPES.fetch(type.to_s) }) + .where("refclassid = 'pg_extension'::pg_catalog.regclass") + end + end + end +end diff --git a/lib/gitlab/database/postgres_foreign_key.rb b/lib/gitlab/database/postgres_foreign_key.rb index 04ef574a451..bb3e1d45f15 100644 --- a/lib/gitlab/database/postgres_foreign_key.rb +++ b/lib/gitlab/database/postgres_foreign_key.rb @@ -5,6 +5,8 @@ module Gitlab class PostgresForeignKey < SharedModel self.primary_key = :oid + has_many :child_foreign_keys, class_name: 'Gitlab::Database::PostgresForeignKey', foreign_key: 'parent_oid' + # These values come from the possible confdeltype / confupdtype values in pg_constraint ACTION_TYPES = { restrict: 'r', @@ -38,6 +40,14 @@ module Gitlab scope :by_constrained_table_name, ->(name) { where(constrained_table_name: name) } + scope :by_constrained_table_name_or_identifier, ->(name) do + if name =~ Database::FULLY_QUALIFIED_IDENTIFIER + by_constrained_table_identifier(name) + else + by_constrained_table_name(name) + end + end + scope :not_inherited, -> { where(is_inherited: false) } scope :by_name, ->(name) { where(name: name) } diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index 36dc6818157..e63c6fc86ea 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -7,6 +7,8 @@ module Gitlab belongs_to :postgres_partitioned_table, foreign_key: 'parent_identifier', primary_key: 'identifier' + # identifier includes the partition schema. + # For example 'gitlab_partitions_static.events_03', or 'gitlab_partitions_dynamic.logs_03' scope :for_identifier, ->(identifier) do unless identifier =~ Gitlab::Database::FULLY_QUALIFIED_IDENTIFIER raise ArgumentError, "Partition name is not fully qualified with a schema: #{identifier}" @@ -19,8 +21,12 @@ module Gitlab for_identifier(identifier).first! end - scope :for_parent_table, ->(name) do - where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) + scope :for_parent_table, ->(parent_table) do + if parent_table =~ Database::FULLY_QUALIFIED_IDENTIFIER + where(parent_identifier: parent_table).order(:name) + else + where("parent_identifier = concat(current_schema(), '.', ?)", parent_table).order(:name) + end end def self.partition_exists?(table_name) diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index b4b9161f0c2..133933b6ccd 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -21,7 +21,7 @@ module Gitlab db_config_name = ::Gitlab::Database.db_config_name(parsed.connection) return unless db_config_name - gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(parsed.pg.tables) + gitlab_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(parsed.pg.tables) return if gitlab_schemas.empty? # to reduce amount of labels sort schemas used diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb index c966ae0e105..976f8c3e7e4 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_validate_connection.rb @@ -15,7 +15,7 @@ module Gitlab def analyze(parsed) tables = parsed.pg.select_tables + parsed.pg.dml_tables - table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + table_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables) return if table_schemas.empty? allowed_schemas = ::Gitlab::Database.gitlab_schemas_for_connection(parsed.connection) diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index 713e1f772e3..d15a0eaa44c 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -22,12 +22,27 @@ module Gitlab self.with_suppressed(false, &blk) end + # This method will temporary ignore the given tables in a current transaction + # This is meant to disable `PreventCrossDB` check for some well known failures + def self.temporary_ignore_tables_in_transaction(tables, url:, &blk) + return yield unless context&.dig(:ignored_tables) + + begin + prev_ignored_tables = context[:ignored_tables] + context[:ignored_tables] = prev_ignored_tables + tables + yield + ensure + context[:ignored_tables] = prev_ignored_tables + end + end + def self.begin! super context.merge!({ transaction_depth_by_db: Hash.new { |h, k| h[k] = 0 }, - modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new } + modified_tables_by_db: Hash.new { |h, k| h[k] = Set.new }, + ignored_tables: [] }) end @@ -57,7 +72,7 @@ module Gitlab if context[:transaction_depth_by_db][database] == 0 context[:modified_tables_by_db][database].clear - # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 + # Attempt to troubleshoot https://gitlab.com/gitlab-org/gitlab/-/issues/351531 ::CrossDatabaseModification::TransactionStackTrackRecord.log_gitlab_transactions_stack(action: :end_of_transaction) elsif context[:transaction_depth_by_db][database] < 0 context[:transaction_depth_by_db][database] = 0 @@ -79,6 +94,9 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/-/issues/343394 tables -= %w[plans gitlab_subscriptions] + # Ignore some tables + tables -= context[:ignored_tables].to_a + return if tables.empty? # All migrations will write to schema_migrations in the same transaction. @@ -88,7 +106,7 @@ module Gitlab context[:modified_tables_by_db][database].merge(tables) all_tables = context[:modified_tables_by_db].values.flat_map(&:to_a) - schemas = ::Gitlab::Database::GitlabSchema.table_schemas(all_tables) + schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(all_tables) schemas += ApplicationRecord.gitlab_transactions_stack @@ -97,10 +115,6 @@ module Gitlab "a transaction modifying the '#{all_tables.to_a.join(", ")}' tables. " \ "Please refer to https://docs.gitlab.com/ee/development/database/multiple_databases.html#removing-cross-database-transactions for details on how to resolve this exception." - if schemas.any? { |s| s.to_s.start_with?("undefined") } - message += " The gitlab_schema was undefined for one or more of the tables in this transaction. Any new tables must be added to lib/gitlab/database/gitlab_schemas.yml ." - end - raise CrossDatabaseModificationAcrossUnsupportedTablesError, message end rescue CrossDatabaseModificationAcrossUnsupportedTablesError => e diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index 4ae3622479f..f3e0fc26946 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -100,7 +100,7 @@ module Gitlab end def dml_schemas(tables) - extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) + extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas!(tables) SCHEMA_MAPPING.each do |schema, mapped_schema| next unless extra_schemas.delete?(schema) diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 78de7161a0f..739e573b6c4 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -28,7 +28,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) Gitlab::Database::AsyncIndexes.drop_pending_indexes! - Gitlab::Database::AsyncForeignKeys.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) + Gitlab::Database::AsyncConstraints.validate_pending_entries! if Feature.enabled?(:database_async_foreign_key_validation, type: :ops) automatic_reindexing end diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb index d81ff4ff1ae..3ae696a71d8 100644 --- a/lib/gitlab/database/schema_helpers.rb +++ b/lib/gitlab/database/schema_helpers.rb @@ -31,8 +31,8 @@ module Gitlab end def trigger_exists?(table_name, name) - connection.select_value(<<~SQL) - SELECT 1 + result = connection.select_value(<<~SQL.squish) + SELECT true FROM pg_catalog.pg_trigger trgr INNER JOIN pg_catalog.pg_class rel ON trgr.tgrelid = rel.oid @@ -42,6 +42,8 @@ module Gitlab AND rel.relname = #{connection.quote(table_name)} AND trgr.tgname = #{connection.quote(name)} SQL + + !!result end def drop_function(name, if_exists: true) diff --git a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb new file mode 100644 index 00000000000..32d638380ea --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + class ColumnDatabaseAdapter + def initialize(query_result) + @query_result = query_result + end + + def name + @name ||= query_result['column_name'] + end + + def table_name + query_result['table_name'] + end + + def data_type + query_result['data_type'] + end + + def default + return unless query_result['column_default'] + + return if name == 'id' || query_result['column_default'].include?('nextval') + + "DEFAULT #{query_result['column_default']}" + end + + def nullable + 'NOT NULL' if query_result['not_null'] + end + + def partition_key? + query_result['partition_key'] + end + + private + + attr_reader :query_result + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb new file mode 100644 index 00000000000..420195d89dd --- /dev/null +++ b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb @@ -0,0 +1,128 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Adapters + UndefinedPGType = Class.new(StandardError) + + class ColumnStructureSqlAdapter + NOT_NULL_CONSTR = :CONSTR_NOTNULL + DEFAULT_CONSTR = :CONSTR_DEFAULT + + MAPPINGS = { + 't' => 'true', + 'f' => 'false' + }.freeze + + attr_reader :table_name + + def initialize(table_name, pg_query_stmt, partitioning_stmt) + @table_name = table_name + @pg_query_stmt = pg_query_stmt + @partitioning_stmt = partitioning_stmt + end + + def name + @name ||= pg_query_stmt.colname + end + + def data_type + type(pg_query_stmt.type_name) + end + + def default + return if name == 'id' + + value = parse_node(constraints.find { |node| node.constraint.contype == DEFAULT_CONSTR }) + + return unless value + + "DEFAULT #{value}" + end + + def nullable + 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR } + end + + def partition_key? + partition_keys.include?(name) + end + + private + + attr_reader :pg_query_stmt, :partitioning_stmt + + def constraints + @constraints ||= pg_query_stmt.constraints + end + + # Returns the node type + # + # pg_type:: type alias, used internally by postgres, +int4+, +int8+, +bool+, +varchar+ + # type:: type name, like +integer+, +bigint+, +boolean+, +character varying+. + # array_ext:: adds the +[]+ extension for array types. + # precision_ext:: adds the precision, if have any, like +(255)+, +(6)+. + # + # @info +timestamp+ and +timestamptz+ have a particular case when precision is defined. + # In this case, the order of the statement needs to be re-arranged from + # timestamp without time zone(6) to timestamp(6) without a time zone. + def type(node) + pg_type = parse_node(node.names.last) + type = PgTypes::TYPES.fetch(pg_type).dup + array_ext = '[]' if node.array_bounds.any? + precision_ext = "(#{node.typmods.map { |typmod| parse_node(typmod) }.join(',')})" if node.typmods.any? + + if %w[timestamp timestamptz].include?(pg_type) + type.gsub!('timestamp', ['timestamp', precision_ext].compact.join('')) + precision_ext = nil + end + + [type, precision_ext, array_ext].compact.join('') + rescue KeyError => exception + raise UndefinedPGType, exception.message + end + + # Parses PGQuery nodes recursively + # + # :constraint:: nodes that groups column default info + # :partition_elem:: node that store partition key info + # :func_cal:: nodes that stores functions, like +now()+ + # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+ + # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+ + # else:: extract node values in the last iteration of the recursion, like +int4+, +1.0+, +now+, +255+ + # + # @note boolean types types are mapped from +t+, +f+ to +true+, +false+ + def parse_node(node) + return unless node + + case node.node + when :constraint + parse_node(node.constraint.raw_expr) + when :partition_elem + node.partition_elem.name + when :func_call + "#{parse_node(node.func_call.funcname.first)}()" + when :a_const + parse_node(node.a_const.val) + when :type_cast + value = parse_node(node.type_cast.arg) + type = type(node.type_cast.type_name) + separator = MAPPINGS.key?(value) ? '' : "::#{type}" + + [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('') + else + node.to_h[node.node].values.last + end + end + + def partition_keys + return [] unless partitioning_stmt + + @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) } + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb index dfc845f0b44..7a0e348a27b 100644 --- a/lib/gitlab/database/schema_validation/database.rb +++ b/lib/gitlab/database/schema_validation/database.rb @@ -4,6 +4,8 @@ module Gitlab module Database module SchemaValidation class Database + STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static' + def initialize(connection) @connection = connection end @@ -12,29 +14,113 @@ module Gitlab index_map[index_name] end + def fetch_trigger_by_name(trigger_name) + trigger_map[trigger_name] + end + + def fetch_table_by_name(table_name) + table_map[table_name] + end + + def index_exists?(index_name) + index_map[index_name].present? + end + + def trigger_exists?(trigger_name) + trigger_map[trigger_name].present? + end + + def table_exists?(table_name) + fetch_table_by_name(table_name).present? + end + def indexes index_map.values end + def triggers + trigger_map.values + end + + def tables + table_map.values + end + private + attr_reader :connection + + def schemas + @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema] + end + def index_map @index_map ||= fetch_indexes.transform_values! do |index_stmt| - Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) + SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) end end - attr_reader :connection + def trigger_map + @trigger_map ||= + fetch_triggers.transform_values! do |trigger_stmt| + SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt) + end + end + + def table_map + @table_map ||= fetch_tables.transform_values! do |stmt| + columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } + + SchemaObjects::Table.new(stmt.first['table_name'], columns) + end + end def fetch_indexes sql = <<~SQL SELECT indexname, indexdef FROM pg_indexes - WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ('public', 'gitlab_partitions_static'); + WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2); + SQL + + connection.select_rows(sql, nil, schemas).to_h + end + + def fetch_triggers + sql = <<~SQL + SELECT triggers.tgname, pg_get_triggerdef(triggers.oid) + FROM pg_catalog.pg_trigger triggers + INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid + INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace + WHERE triggers.tgisinternal IS FALSE + AND nsp.nspname IN ($1, $2) + SQL + + connection.select_rows(sql, nil, schemas).to_h + end + + def fetch_tables + sql = <<~SQL + SELECT + table_information.relname AS table_name, + col_information.attname AS column_name, + col_information.attnotnull AS not_null, + col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key, + format_type(col_information.atttypid, col_information.atttypmod) AS data_type, + pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default + FROM pg_attribute AS col_information + JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid + JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid + LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid + LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid + AND col_information.attnum = col_default_information.adnum + WHERE NOT col_information.attisdropped + AND col_information.attnum > 0 + AND table_information.relkind IN ('r', 'p') + AND schema_information.nspname IN ($1, $2) SQL - @fetch_indexes ||= connection.exec_query(sql).rows.to_h + connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } end end end diff --git a/lib/gitlab/database/schema_validation/inconsistency.rb b/lib/gitlab/database/schema_validation/inconsistency.rb new file mode 100644 index 00000000000..766f48ef339 --- /dev/null +++ b/lib/gitlab/database/schema_validation/inconsistency.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Inconsistency + def initialize(validator_class, structure_sql_object, database_object) + @validator_class = validator_class + @structure_sql_object = structure_sql_object + @database_object = database_object + end + + def error_message + format(validator_class::ERROR_MESSAGE, object_name) + end + + def type + validator_class.name.demodulize.underscore + end + + def object_type + structure_sql_object&.class&.name&.demodulize || database_object&.class&.name&.demodulize + end + + def table_name + structure_sql_object&.table_name || database_object&.table_name + end + + def object_name + structure_sql_object&.name || database_object&.name + end + + def diff + Diffy::Diff.new(structure_sql_statement, database_statement) + end + + def inspect + <<~MSG + #{'-' * 54} + #{error_message} + Diff: + #{diff.to_s(:color)} + #{'-' * 54} + MSG + end + + def structure_sql_statement + return unless structure_sql_object + + "#{structure_sql_object.statement}\n" + end + + def database_statement + return unless database_object + + "#{database_object.statement}\n" + end + + private + + attr_reader :validator_class, :structure_sql_object, :database_object + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/inconsistency_filter.rb b/lib/gitlab/database/schema_validation/inconsistency_filter.rb new file mode 100644 index 00000000000..aa3a71c0edb --- /dev/null +++ b/lib/gitlab/database/schema_validation/inconsistency_filter.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class InconsistencyFilter + def initialize(tables, triggers) + @tables = tables + @triggers = triggers + end + + def to_proc + proc do |inconsistency| + inconsistency unless ignored?(inconsistency) + end + end + + private + + attr_reader :tables, :triggers + + def ignored?(inconsistency) + case inconsistency.type + in 'extra_tables' | 'missing_tables' + ignored_table?(inconsistency.table_name) + in 'extra_triggers' | 'missing_triggers' + ignored_trigger?(inconsistency.object_name) + else + false + end + end + + def ignored_table?(name) + tables.include?(name) + end + + def ignored_trigger?(name) + triggers.any? { |ignored_object| name.to_s.include?(ignored_object) } + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/index.rb b/lib/gitlab/database/schema_validation/index.rb deleted file mode 100644 index af0d5f31f4e..00000000000 --- a/lib/gitlab/database/schema_validation/index.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Index - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - parsed_stmt.idxname - end - - def statement - @statement ||= PgQuery.deparse_stmt(parsed_stmt) - end - - private - - attr_reader :parsed_stmt - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/indexes.rb b/lib/gitlab/database/schema_validation/indexes.rb deleted file mode 100644 index b7c3705bde9..00000000000 --- a/lib/gitlab/database/schema_validation/indexes.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Indexes - def initialize(structure_sql, database) - @structure_sql = structure_sql - @database = database - end - - def missing_indexes - structure_sql.indexes.map(&:name) - database.indexes.map(&:name) - end - - def extra_indexes - database.indexes.map(&:name) - structure_sql.indexes.map(&:name) - end - - def wrong_indexes - structure_sql.indexes.filter_map do |structure_sql_index| - database_index = database.fetch_index_by_name(structure_sql_index.name) - - next if database_index.nil? - next if database_index.statement == structure_sql_index.statement - - structure_sql_index.name - end - end - - private - - attr_reader :structure_sql, :database - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/pg_types.rb b/lib/gitlab/database/schema_validation/pg_types.rb new file mode 100644 index 00000000000..0a1999d056e --- /dev/null +++ b/lib/gitlab/database/schema_validation/pg_types.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class PgTypes + TYPES = { + 'bool' => 'boolean', + 'bytea' => 'bytea', + 'char' => '"char"', + 'int8' => 'bigint', + 'int2' => 'smallint', + 'int4' => 'integer', + 'regproc' => 'regproc', + 'text' => 'text', + 'oid' => 'oid', + 'tid' => 'tid', + 'xid' => 'xid', + 'cid' => 'cid', + 'json' => 'json', + 'xml' => 'xml', + 'pg_node_tree' => 'pg_node_tree', + 'pg_ndistinct' => 'pg_ndistinct', + 'pg_dependencies' => 'pg_dependencies', + 'pg_mcv_list' => 'pg_mcv_list', + 'xid8' => 'xid8', + 'path' => 'path', + 'polygon' => 'polygon', + 'float4' => 'real', + 'float8' => 'double precision', + 'circle' => 'circle', + 'money' => 'money', + 'macaddr' => 'macaddr', + 'inet' => 'inet', + 'cidr' => 'cidr', + 'macaddr8' => 'macaddr8', + 'aclitem' => 'aclitem', + 'bpchar' => 'character', + 'varchar' => 'character varying', + 'date' => 'date', + 'time' => 'time without time zone', + 'timestamp' => 'timestamp without time zone', + 'timestamptz' => 'timestamp with time zone', + 'interval' => 'interval', + 'timetz' => 'time with time zone', + 'bit' => 'bit', + 'varbit' => 'bit varying', + 'numeric' => 'numeric', + 'refcursor' => 'refcursor', + 'regprocedure' => 'regprocedure', + 'regoper' => 'regoper', + 'regoperator' => 'regoperator', + 'regclass' => 'regclass', + 'regcollation' => 'regcollation', + 'regtype' => 'regtype', + 'regrole' => 'regrole', + 'regnamespace' => 'regnamespace', + 'uuid' => 'uuid', + 'pg_lsn' => 'pg_lsn', + 'tsvector' => 'tsvector', + 'gtsvector' => 'gtsvector', + 'tsquery' => 'tsquery', + 'regconfig' => 'regconfig', + 'regdictionary' => 'regdictionary', + 'jsonb' => 'jsonb', + 'jsonpath' => 'jsonpath', + 'txid_snapshot' => 'txid_snapshot', + 'pg_snapshot' => 'pg_snapshot' + }.freeze + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/runner.rb b/lib/gitlab/database/schema_validation/runner.rb new file mode 100644 index 00000000000..7a02c8a16d6 --- /dev/null +++ b/lib/gitlab/database/schema_validation/runner.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class Runner + def initialize(structure_sql, database, validators: Validators::BaseValidator.all_validators) + @structure_sql = structure_sql + @database = database + @validators = validators + end + + def execute + validators.flat_map { |c| c.new(structure_sql, database).execute } + end + + private + + attr_reader :structure_sql, :database, :validators + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_inconsistency.rb b/lib/gitlab/database/schema_validation/schema_inconsistency.rb new file mode 100644 index 00000000000..6f50603e784 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_inconsistency.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class SchemaInconsistency < ApplicationRecord + self.table_name = :schema_inconsistencies + + belongs_to :issue + + validates :object_name, :valitador_name, :table_name, presence: true + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb new file mode 100644 index 00000000000..43d30dc54ae --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/base.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Base + def initialize(parsed_stmt) + @parsed_stmt = parsed_stmt + end + + def name + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + def table_name + parsed_stmt.relation.relname + end + + def statement + @statement ||= PgQuery.deparse_stmt(parsed_stmt) + end + + private + + attr_reader :parsed_stmt + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/column.rb b/lib/gitlab/database/schema_validation/schema_objects/column.rb new file mode 100644 index 00000000000..bd219300a13 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/column.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Column + def initialize(adapter) + @adapter = adapter + end + + attr_reader :adapter + + delegate :name, :table_name, :partition_key?, to: :adapter + + def statement + [name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ') + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb new file mode 100644 index 00000000000..28d61b18266 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/index.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Index < Base + def name + parsed_stmt.idxname + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/table.rb b/lib/gitlab/database/schema_validation/schema_objects/table.rb new file mode 100644 index 00000000000..de2e675d20e --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/table.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Table + def initialize(name, columns) + @name = name + @columns = columns + end + + attr_reader :name, :columns + + def table_name + name + end + + def statement + format('CREATE TABLE %s (%s)', name, columns_statement) + end + + def fetch_column_by_name(column_name) + columns.find { |column| column.name == column_name } + end + + def column_exists?(column_name) + fetch_column_by_name(column_name).present? + end + + private + + def columns_statement + columns.reject(&:partition_key?).map(&:statement).join(', ') + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb new file mode 100644 index 00000000000..508e6b27ed3 --- /dev/null +++ b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module SchemaObjects + class Trigger < Base + def name + parsed_stmt.trigname + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb index 32c69a0e5e7..299db1c2a7e 100644 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ b/lib/gitlab/database/schema_validation/structure_sql.rb @@ -4,29 +4,82 @@ module Gitlab module Database module SchemaValidation class StructureSql - def initialize(structure_file_path) + DEFAULT_SCHEMA = 'public' + + def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA) @structure_file_path = structure_file_path + @schema_name = schema_name + end + + def index_exists?(index_name) + indexes.find { |index| index.name == index_name }.present? + end + + def trigger_exists?(trigger_name) + triggers.find { |trigger| trigger.name == trigger_name }.present? + end + + def fetch_table_by_name(table_name) + tables.find { |table| table.name == table_name } + end + + def table_exists?(table_name) + fetch_table_by_name(table_name).present? end def indexes - @indexes ||= index_statements.map do |index_statement| - index_statement.relation.schemaname = "public" if index_statement.relation.schemaname == '' + @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index) + end + + def triggers + @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) + end - Index.new(index_statement) + def tables + @tables ||= table_statements.map do |stmt| + table_name = stmt.relation.relname + partition_stmt = stmt.partspec + + columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column| + adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt) + SchemaObjects::Column.new(adapter) + end + + SchemaObjects::Table.new(table_name, columns) end end private - attr_reader :structure_file_path + attr_reader :structure_file_path, :schema_name def index_statements - parsed_structure_file.tree.stmts.filter_map { |s| s.stmt.index_stmt } + statements.filter_map { |s| s.stmt.index_stmt } + end + + def trigger_statements + statements.filter_map { |s| s.stmt.create_trig_stmt } + end + + def table_statements + statements.filter_map { |s| s.stmt.create_stmt } + end + + def statements + @statements ||= parsed_structure_file.tree.stmts end def parsed_structure_file PgQuery.parse(File.read(structure_file_path)) end + + def map_with_default_schema(statements, validation_class) + statements.map do |statement| + statement.relation.schemaname = schema_name if statement.relation.schemaname == '' + + validation_class.new(statement) + end + end end end end diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb new file mode 100644 index 00000000000..47c3492971c --- /dev/null +++ b/lib/gitlab/database/schema_validation/track_inconsistency.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + class TrackInconsistency + def initialize(inconsistency, project, user) + @inconsistency = inconsistency + @project = project + @user = user + end + + def execute + return unless Gitlab.com? + return if inconsistency_record.present? + + result = ::Issues::CreateService.new(container: project, current_user: user, params: params, + spam_params: nil).execute + + track_inconsistency(result[:issue]) if result.success? + end + + private + + attr_reader :inconsistency, :project, :user + + def track_inconsistency(issue) + schema_inconsistency_model.create( + issue: issue, + object_name: inconsistency.object_name, + table_name: inconsistency.table_name, + valitador_name: inconsistency.type + ) + end + + def params + { + title: issue_title, + description: issue_description, + issue_type: 'issue', + labels: %w[database database-inconsistency-report] + } + end + + def issue_title + "New schema inconsistency: #{inconsistency.object_name}" + end + + def issue_description + <<~MSG + We have detected a new schema inconsistency. + + **Table name:** #{inconsistency.table_name}\ + **Object name:** #{inconsistency.object_name}\ + **Validator name:** #{inconsistency.type}\ + **Object type:** #{inconsistency.object_type}\ + **Error message:** #{inconsistency.error_message} + + + **Structure.sql statement:** + + ```sql + #{inconsistency.structure_sql_statement} + ``` + + **Database statement:** + + ```sql + #{inconsistency.database_statement} + ``` + + **Diff:** + + ```diff + #{inconsistency.diff} + + ``` + + + For more information, please contact the database team. + MSG + end + + def schema_inconsistency_model + Gitlab::Database::SchemaValidation::SchemaInconsistency + end + + def inconsistency_record + schema_inconsistency_model.find_by( + object_name: inconsistency.object_name, + table_name: inconsistency.table_name, + valitador_name: inconsistency.type + ) + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb new file mode 100644 index 00000000000..58e0bf5292b --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/base_validator.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class BaseValidator + ERROR_MESSAGE = 'A schema inconsistency has been found' + + def initialize(structure_sql, database) + @structure_sql = structure_sql + @database = database + end + + def self.all_validators + [ + ExtraTables, + ExtraTableColumns, + ExtraIndexes, + ExtraTriggers, + MissingTables, + MissingTableColumns, + MissingIndexes, + MissingTriggers, + DifferentDefinitionTables, + DifferentDefinitionIndexes, + DifferentDefinitionTriggers + ] + end + + def execute + raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" + end + + private + + attr_reader :structure_sql, :database + + def build_inconsistency(validator_class, structure_sql_object, database_object) + Inconsistency.new(validator_class, structure_sql_object, database_object) + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb new file mode 100644 index 00000000000..ba12b3cdc61 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionIndexes < BaseValidator + ERROR_MESSAGE = "The %s index has a different statement between structure.sql and database" + + def execute + structure_sql.indexes.filter_map do |structure_sql_index| + database_index = database.fetch_index_by_name(structure_sql_index.name) + + next if database_index.nil? + next if database_index.statement == structure_sql_index.statement + + build_inconsistency(self.class, structure_sql_index, database_index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb new file mode 100644 index 00000000000..9fbddbd3fcd --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionTables < BaseValidator + ERROR_MESSAGE = "The table %s has a different column statement between structure.sql and database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + table_name = structure_sql_table.name + database_table = database.fetch_table_by_name(table_name) + + next unless database_table + + db_diffs, structure_diffs = column_diffs(database_table, structure_sql_table.columns) + + if db_diffs.any? + build_inconsistency(self.class, + SchemaObjects::Table.new(table_name, db_diffs), + SchemaObjects::Table.new(table_name, structure_diffs)) + end + end + end + + private + + def column_diffs(db_table, columns) + db_diffs = [] + structure_diffs = [] + + columns.each do |column| + db_column = db_table.fetch_column_by_name(column.name) + + next unless db_column + + next if db_column.statement == column.statement + + db_diffs << db_column + structure_diffs << column + end + + [db_diffs, structure_diffs] + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb new file mode 100644 index 00000000000..79ffe9a6a98 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class DifferentDefinitionTriggers < BaseValidator + ERROR_MESSAGE = "The %s trigger has a different statement between structure.sql and database" + + def execute + structure_sql.triggers.filter_map do |structure_sql_trigger| + database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name) + + next if database_trigger.nil? + next if database_trigger.statement == structure_sql_trigger.statement + + build_inconsistency(self.class, structure_sql_trigger, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb new file mode 100644 index 00000000000..c8d3749894b --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraIndexes < BaseValidator + ERROR_MESSAGE = "The index %s is present in the database, but not in the structure.sql file" + + def execute + database.indexes.filter_map do |database_index| + next if structure_sql.index_exists?(database_index.name) + + build_inconsistency(self.class, nil, database_index) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb new file mode 100644 index 00000000000..823b01cf808 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraTableColumns < BaseValidator + ERROR_MESSAGE = "The table %s has columns present in the database, but not in the structure.sql file" + + def execute + database.tables.filter_map do |database_table| + table_name = database_table.name + structure_sql_table = structure_sql.fetch_table_by_name(table_name) + + next unless structure_sql_table + + inconsistencies = database_table.columns.filter_map do |database_table_column| + next if structure_sql_table.column_exists?(database_table_column.name) + + database_table_column + end + + if inconsistencies.any? + build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_tables.rb b/lib/gitlab/database/schema_validation/validators/extra_tables.rb new file mode 100644 index 00000000000..99e98eb8f67 --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_tables.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraTables < BaseValidator + ERROR_MESSAGE = "The table %s is present in the database, but not in the structure.sql file" + + def execute + database.tables.filter_map do |database_table| + next if structure_sql.table_exists?(database_table.name) + + build_inconsistency(self.class, nil, database_table) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb new file mode 100644 index 00000000000..37dcbc53e2e --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class ExtraTriggers < BaseValidator + ERROR_MESSAGE = "The trigger %s is present in the database, but not in the structure.sql file" + + def execute + database.triggers.filter_map do |database_trigger| + next if structure_sql.trigger_exists?(database_trigger.name) + + build_inconsistency(self.class, nil, database_trigger) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb new file mode 100644 index 00000000000..7f81aaccf0f --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingIndexes < BaseValidator + ERROR_MESSAGE = "The index %s is missing from the database" + + def execute + structure_sql.indexes.filter_map do |structure_sql_index| + next if database.index_exists?(structure_sql_index.name) + + build_inconsistency(self.class, structure_sql_index, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb new file mode 100644 index 00000000000..b49d53823ee --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingTableColumns < BaseValidator + ERROR_MESSAGE = "The table %s has columns missing from the database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + table_name = structure_sql_table.name + database_table = database.fetch_table_by_name(table_name) + + next unless database_table + + inconsistencies = structure_sql_table.columns.filter_map do |structure_table_column| + next if database_table.column_exists?(structure_table_column.name) + + structure_table_column + end + + if inconsistencies.any? + build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_tables.rb b/lib/gitlab/database/schema_validation/validators/missing_tables.rb new file mode 100644 index 00000000000..f1c9383487d --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_tables.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingTables < BaseValidator + ERROR_MESSAGE = "The table %s is missing from the database" + + def execute + structure_sql.tables.filter_map do |structure_sql_table| + next if database.table_exists?(structure_sql_table.name) + + build_inconsistency(self.class, structure_sql_table, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb new file mode 100644 index 00000000000..36236463bbf --- /dev/null +++ b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module SchemaValidation + module Validators + class MissingTriggers < BaseValidator + ERROR_MESSAGE = "The trigger %s is missing from the database" + + def execute + structure_sql.triggers.filter_map do |structure_sql_trigger| + next if database.trigger_exists?(structure_sql_trigger.name) + + build_inconsistency(self.class, structure_sql_trigger, nil) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index c417ce716e8..0b0d46f4b0e 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -3,11 +3,12 @@ module Gitlab module Database class TablesLocker - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze def initialize(logger: nil, dry_run: false) @logger = logger @dry_run = dry_run + @result = [] end def unlock_writes @@ -16,11 +17,15 @@ module Gitlab # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE - lock_writes_manager(table_name, connection, database_name).unlock_writes + unlock_writes_on_table(table_name, connection, database_name) end end + + @result end + # It locks the tables on the database where they don't belong. Also it unlocks the tables + # on the database where they belong def lock_writes Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) @@ -30,16 +35,36 @@ module Gitlab next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE if schemas_for_connection.include?(schema_name) - lock_writes_manager(table_name, connection, database_name).unlock_writes + unlock_writes_on_table(table_name, connection, database_name) else - lock_writes_manager(table_name, connection, database_name).lock_writes + lock_writes_on_table(table_name, connection, database_name) end end end + + @result end private + # Unlocks the writes on the table and its partitions + def unlock_writes_on_table(table_name, connection, database_name) + @result << lock_writes_manager(table_name, connection, database_name).unlock_writes + + table_attached_partitions(table_name, connection) do |postgres_partition| + @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).unlock_writes + end + end + + # It locks the writes on the table and its partitions + def lock_writes_on_table(table_name, connection, database_name) + @result << lock_writes_manager(table_name, connection, database_name).lock_writes + + table_attached_partitions(table_name, connection) do |postgres_partition| + @result << lock_writes_manager(postgres_partition.identifier, connection, database_name).lock_writes + end + end + def tables_to_lock(connection, &block) Gitlab::Database::GitlabSchema.tables_to_schema.each(&block) @@ -50,6 +75,14 @@ module Gitlab end end + def table_attached_partitions(table_name, connection, &block) + Gitlab::Database::SharedModel.using_connection(connection) do + break unless Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) + + Gitlab::Database::PostgresPartitionedTable.each_partition(table_name, &block) + end + end + def lock_writes_manager(table_name, connection, database_name) Gitlab::Database::LockWritesManager.new( table_name: table_name, diff --git a/lib/gitlab/database_importers/instance_administrators/create_group.rb b/lib/gitlab/database_importers/instance_administrators/create_group.rb deleted file mode 100644 index bb489ced3d2..00000000000 --- a/lib/gitlab/database_importers/instance_administrators/create_group.rb +++ /dev/null @@ -1,133 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module InstanceAdministrators - class CreateGroup < ::BaseService - include Stepable - - NAME = 'GitLab Instance' - PATH_PREFIX = 'gitlab-instance' - VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL - - steps :validate_application_settings, - :validate_admins, - :create_group, - :save_group_id, - :add_group_members, - :track_event - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_application_settings(result) - return success(result) if application_settings - - log_error('No application_settings found') - error(_('No application_settings found')) - end - - def validate_admins(result) - unless instance_admins.any? - log_error('No active admin user found') - return error(_('No active admin user found')) - end - - success(result) - end - - def create_group(result) - if group_created? - log_info(_('Instance administrators group already exists')) - result[:group] = instance_administrators_group - return success(result) - end - - result[:group] = ::Groups::CreateService.new(instance_admins.first, create_group_params).execute - - if result[:group].persisted? - success(result) - else - log_error("Could not create instance administrators group. Errors: %{errors}" % { errors: result[:group].errors.full_messages }) - error(_('Could not create group')) - end - end - - def save_group_id(result) - return success(result) if group_created? - - response = application_settings.update( - instance_administrators_group_id: result[:group].id - ) - - if response - success(result) - else - log_error("Could not save instance administrators group ID, errors: %{errors}" % { errors: application_settings.errors.full_messages }) - error(_('Could not save group ID')) - end - end - - def add_group_members(result) - group = result[:group] - members = group.add_members(members_to_add(group), Gitlab::Access::MAINTAINER) - errors = members.flat_map { |member| member.errors.full_messages } - - if errors.any? - log_error('Could not add admins as members to self-monitoring project. Errors: %{errors}' % { errors: errors }) - error(_('Could not add admins as members')) - else - success(result) - end - end - - def track_event(result) - ::Gitlab::Tracking.event("instance_administrators_group", "group_created", namespace: result[:group]) - - success(result) - end - - def group_created? - instance_administrators_group.present? - end - - def application_settings - @application_settings ||= ApplicationSetting.current_without_cache - end - - def instance_administrators_group - application_settings.instance_administrators_group - end - - def instance_admins - @instance_admins ||= User.admins.active - end - - def members_to_add(group) - # Exclude admins who are already members of group because - # `group.add_members(users)` returns an error if the users parameter contains - # users who are already members of the group. - instance_admins - group.members.collect(&:user) - end - - def create_group_params - { - name: NAME, - visibility_level: VISIBILITY_LEVEL, - - # The 8 random characters at the end are so that the path does not - # clash with any existing group that the user might have created. - path: "#{PATH_PREFIX}-#{SecureRandom.hex(4)}" - } - end - end - end - end -end diff --git a/lib/gitlab/database_importers/security/training_providers/importer.rb b/lib/gitlab/database_importers/security/training_providers/importer.rb index aa6a9f29c6d..87bef6400fa 100644 --- a/lib/gitlab/database_importers/security/training_providers/importer.rb +++ b/lib/gitlab/database_importers/security/training_providers/importer.rb @@ -20,6 +20,13 @@ module Gitlab url: "https://integration-api.securecodewarrior.com/api/v1/trial" }.freeze + SECUREFLAG_DATA = { + name: 'SecureFlag', + description: "Get remediation advice with example code and recommended hands-on labs in a fully + interactive virtualised environment.", + url: "https://knowledge-base-api.secureflag.com/gitlab" + }.freeze + module Security class TrainingProvider < ApplicationRecord self.table_name = 'security_training_providers' @@ -31,7 +38,7 @@ module Gitlab timestamps = { created_at: current_time, updated_at: current_time } Security::TrainingProvider.upsert_all( - [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps)], + [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps), SECUREFLAG_DATA.merge(timestamps)], unique_by: :index_security_training_providers_on_unique_name ) end diff --git a/lib/gitlab/database_importers/self_monitoring/helpers.rb b/lib/gitlab/database_importers/self_monitoring/helpers.rb deleted file mode 100644 index 6956401e20d..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/helpers.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Helpers - def application_settings - @application_settings ||= ApplicationSetting.current_without_cache - end - - def project_created? - self_monitoring_project.present? - end - - def self_monitoring_project - application_settings.self_monitoring_project - end - - def self_monitoring_project_id - application_settings.self_monitoring_project_id - end - end - end - end -end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb deleted file mode 100644 index be500171bef..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ /dev/null @@ -1,171 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Project - class CreateService < ::BaseService - include Stepable - include SelfMonitoring::Helpers - - VISIBILITY_LEVEL = Gitlab::VisibilityLevel::INTERNAL - PROJECT_NAME = 'Monitoring' - - steps :validate_application_settings, - :create_group, - :create_project, - :save_project_id, - :create_environment, - :add_prometheus_manual_configuration, - :track_event - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_application_settings(_result) - return success if application_settings - - log_error('No application_settings found') - error(_('No application_settings found')) - end - - def create_group(result) - create_group_response = - Gitlab::DatabaseImporters::InstanceAdministrators::CreateGroup.new.execute - - if create_group_response[:status] == :success - success(result.merge(create_group_response)) - else - error(create_group_response[:message]) - end - end - - def create_project(result) - if project_created? - log_info('Instance administration project already exists') - result[:project] = self_monitoring_project - return success(result) - end - - owner = result[:group].owners.first - - result[:project] = ::Projects::CreateService.new(owner, create_project_params(result[:group])).execute - - if result[:project].persisted? - success(result) - else - log_error("Could not create instance administration project. Errors: %{errors}" % { errors: result[:project].errors.full_messages }) - error(_('Could not create project')) - end - end - - def save_project_id(result) - return success(result) if project_created? - - response = application_settings.update( - self_monitoring_project_id: result[:project].id - ) - - if response - # In the add_prometheus_manual_configuration method, the Prometheus - # server_address config is saved as an api_url in the Integrations::Prometheus - # model. There are validates hooks in the Integrations::Prometheus model that - # check if the project associated with the Integrations::Prometheus is the - # self_monitoring project. It checks - # Gitlab::CurrentSettings.self_monitoring_project_id, which is why the - # Gitlab::CurrentSettings cache needs to be expired here, so that - # Integrations::Prometheus sees the latest self_monitoring_project_id. - Gitlab::CurrentSettings.expire_current_application_settings - success(result) - else - log_error("Could not save instance administration project ID, errors: %{errors}" % { errors: application_settings.errors.full_messages }) - error(_('Could not save project ID')) - end - end - - def create_environment(result) - return success(result) if result[:project].environments.exists? - - environment = ::Environment.new(project_id: result[:project].id, name: 'production') - - if environment.save - success(result) - else - log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) - error(_('Could not create environment')) - end - end - - def add_prometheus_manual_configuration(result) - return success(result) unless prometheus_enabled? - return success(result) unless prometheus_server_address.present? - - prometheus = result[:project].find_or_initialize_integration('prometheus') - - unless prometheus.update(prometheus_integration_attributes) - log_error('Could not save prometheus manual configuration for self-monitoring project. Errors: %{errors}' % { errors: prometheus.errors.full_messages }) - return error(_('Could not save prometheus manual configuration')) - end - - success(result) - end - - def track_event(result) - project = result[:project] - ::Gitlab::Tracking.event("self_monitoring", "project_created", project: project, namespace: project.namespace) - - success(result) - end - - def parse_url(uri_string) - Addressable::URI.parse(uri_string) - rescue Addressable::URI::InvalidURIError, TypeError - end - - def prometheus_enabled? - ::Gitlab::Prometheus::Internal.prometheus_enabled? - end - - def prometheus_server_address - ::Gitlab::Prometheus::Internal.server_address - end - - def docs_path - Rails.application.routes.url_helpers.help_page_path( - 'administration/monitoring/gitlab_self_monitoring_project/index' - ) - end - - def create_project_params(group) - { - initialize_with_readme: true, - visibility_level: VISIBILITY_LEVEL, - name: PROJECT_NAME, - description: "This project is automatically generated and helps monitor this GitLab instance. [Learn more](#{docs_path}).", - namespace_id: group.id - } - end - - def internal_prometheus_server_address_uri - ::Gitlab::Prometheus::Internal.uri - end - - def prometheus_integration_attributes - { - api_url: internal_prometheus_server_address_uri, - manual_configuration: true, - active: true - } - end - end - end - end - end -end diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb deleted file mode 100644 index d5bed94d735..00000000000 --- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DatabaseImporters - module SelfMonitoring - module Project - class DeleteService < ::BaseService - include Stepable - include SelfMonitoring::Helpers - - steps :validate_self_monitoring_project_exists, - :destroy_project - - def initialize - super(nil) - end - - def execute - execute_steps - end - - private - - def validate_self_monitoring_project_exists(result) - unless project_created? || self_monitoring_project_id.present? - return error(_('Self-monitoring project does not exist')) - end - - success(result) - end - - def destroy_project(result) - return success(result) unless project_created? - - if self_monitoring_project.destroy - success(result) - else - log_error(self_monitoring_project.errors.full_messages) - error(_('Error deleting project. Check logs for error details.')) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database_importers/work_items/base_type_importer.rb b/lib/gitlab/database_importers/work_items/base_type_importer.rb index 9796a5905e3..8f8f44e8392 100644 --- a/lib/gitlab/database_importers/work_items/base_type_importer.rb +++ b/lib/gitlab/database_importers/work_items/base_type_importer.rb @@ -18,7 +18,10 @@ module Gitlab progress: 'Progress', status: 'Status', requirement_legacy: 'Requirement legacy', - test_reports: 'Test reports' + test_reports: 'Test reports', + notifications: 'Notifications', + current_user_todos: "Current user todos", + award_emoji: 'Award emoji' }.freeze WIDGETS_FOR_TYPE = { @@ -32,23 +35,36 @@ module Gitlab :notes, :iteration, :weight, - :health_status + :health_status, + :notifications, + :current_user_todos, + :award_emoji ], incident: [ + :assignees, :description, :hierarchy, - :notes + :notes, + :notifications, + :current_user_todos, + :award_emoji ], test_case: [ :description, - :notes + :notes, + :notifications, + :current_user_todos, + :award_emoji ], requirement: [ :description, :notes, :status, :requirement_legacy, - :test_reports + :test_reports, + :notifications, + :current_user_todos, + :award_emoji ], task: [ :assignees, @@ -59,7 +75,10 @@ module Gitlab :milestone, :notes, :iteration, - :weight + :weight, + :notifications, + :current_user_todos, + :award_emoji ], objective: [ :assignees, @@ -69,7 +88,10 @@ module Gitlab :milestone, :notes, :health_status, - :progress + :progress, + :notifications, + :current_user_todos, + :award_emoji ], key_result: [ :assignees, @@ -79,7 +101,10 @@ module Gitlab :start_and_due_date, :notes, :health_status, - :progress + :progress, + :notifications, + :current_user_todos, + :award_emoji ] }.freeze diff --git a/lib/gitlab/deprecation_json_logger.rb b/lib/gitlab/deprecation_json_logger.rb index 9796b24868b..5b0900a86dd 100644 --- a/lib/gitlab/deprecation_json_logger.rb +++ b/lib/gitlab/deprecation_json_logger.rb @@ -2,6 +2,8 @@ module Gitlab class DeprecationJsonLogger < Gitlab::JsonLogger + exclude_context! + def self.file_name_noext 'deprecation_json' end diff --git a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml index 95f15bd6dee..fe1baeb7b67 100644 --- a/lib/gitlab/design_management/copy_design_collection_model_attributes.yml +++ b/lib/gitlab/design_management/copy_design_collection_model_attributes.yml @@ -16,6 +16,7 @@ design_attributes: - filename - relative_position + - description version_attributes: - author_id @@ -30,6 +31,8 @@ ignore_design_attributes: - issue_id - project_id - iid + - description_html + - cached_markdown_version ignore_version_attributes: - id diff --git a/lib/gitlab/diff/highlight.rb b/lib/gitlab/diff/highlight.rb index 225b4f7cf86..95ea3fe9f0f 100644 --- a/lib/gitlab/diff/highlight.rb +++ b/lib/gitlab/diff/highlight.rb @@ -24,15 +24,15 @@ module Gitlab end def highlight - populate_marker_ranges if Feature.enabled?(:use_marker_ranges, project) + populate_marker_ranges - @diff_lines.map.with_index do |diff_line, index| + @diff_lines.map do |diff_line| diff_line = diff_line.dup # ignore highlighting for "match" lines next diff_line if diff_line.meta? rich_line = apply_syntax_highlight(diff_line) - rich_line = apply_marker_ranges_highlight(diff_line, rich_line, index) + rich_line = apply_marker_ranges_highlight(diff_line, rich_line) diff_line.rich_text = rich_line @@ -60,12 +60,8 @@ module Gitlab highlight_line(diff_line) || ERB::Util.html_escape(diff_line.text) end - def apply_marker_ranges_highlight(diff_line, rich_line, index) - marker_ranges = if Feature.enabled?(:use_marker_ranges, project) - diff_line.marker_ranges - else - inline_diffs[index] - end + def apply_marker_ranges_highlight(diff_line, rich_line) + marker_ranges = diff_line.marker_ranges return rich_line if marker_ranges.blank? @@ -134,12 +130,6 @@ module Gitlab end end - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 - # ------------------------------------------------------------------------ - def inline_diffs - @inline_diffs ||= InlineDiff.for_lines(@raw_lines) - end - def old_lines @old_lines ||= highlighted_blob_lines(diff_file.old_blob) end diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 5128b09aef4..63a437b021d 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -71,7 +71,6 @@ module Gitlab strong_memoize(:redis_key) do options = [ diff_options, - Feature.enabled?(:use_marker_ranges, diffable.project), Feature.enabled?(:diff_line_syntax_highlighting, diffable.project) ] options_for_key = OpenSSL::Digest::SHA256.hexdigest(options.join) diff --git a/lib/gitlab/diff/inline_diff.rb b/lib/gitlab/diff/inline_diff.rb index 802da50cfc6..7f760a23f45 100644 --- a/lib/gitlab/diff/inline_diff.rb +++ b/lib/gitlab/diff/inline_diff.rb @@ -17,27 +17,6 @@ module Gitlab CharDiff.new(old_line, new_line).changed_ranges(offset: offset) end - - # Deprecated: https://gitlab.com/gitlab-org/gitlab/-/issues/324638 - class << self - def for_lines(lines) - pair_selector = Gitlab::Diff::PairSelector.new(lines) - - inline_diffs = [] - - pair_selector.each do |old_index, new_index| - old_line = lines[old_index] - new_line = lines[new_index] - - old_diffs, new_diffs = new(old_line, new_line, offset: 1).inline_diffs - - inline_diffs[old_index] = old_diffs - inline_diffs[new_index] = new_diffs - end - - inline_diffs - end - end end end end diff --git a/lib/gitlab/diff/parser.rb b/lib/gitlab/diff/parser.rb index b29c75ed467..a506bc3aaa2 100644 --- a/lib/gitlab/diff/parser.rb +++ b/lib/gitlab/diff/parser.rb @@ -84,8 +84,6 @@ module Gitlab "new" when "-" "old" - else - nil end end end diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 14cb773251b..df93e6e91b4 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -16,11 +16,11 @@ module Gitlab def write_multiple(mapping) with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| + redis.pipelined do |pipelined| mapping.each do |raw_key, value| key = cache_key_for(raw_key) - multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) + pipelined.set(key, gzip_compress(value.to_json), ex: EXPIRATION) end end end @@ -41,7 +41,13 @@ module Gitlab content = with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.mget(keys) + if ::Feature.enabled?(:use_pipeline_over_multikey) + redis.pipelined do |pipeline| + keys.each { |key| pipeline.get(key) } + end + else + redis.mget(keys) + end end end @@ -66,7 +72,13 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.del(keys) + if ::Feature.enabled?(:use_pipeline_over_multikey) + redis.pipelined do |pipeline| + keys.each { |key| pipeline.del(key) } + end.sum + else + redis.del(keys) + end end end end diff --git a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb index e0884557496..0624fe934f9 100644 --- a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb +++ b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb @@ -10,9 +10,7 @@ module Gitlab # additional security. SALT = '' - def self.transform_secret(plain_secret, stored_as_hash = false) - return plain_secret if Feature.disabled?(:hash_oauth_secrets) && !stored_as_hash - + def self.transform_secret(plain_secret) Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) end @@ -28,8 +26,7 @@ module Gitlab # Securely compare the given +input+ value with a +stored+ value # processed by +transform_secret+. def self.secret_matches?(input, stored) - stored_as_hash = stored.starts_with?('$pbkdf2-') - transformed_input = transform_secret(input, stored_as_hash) + transformed_input = transform_secret(input) ActiveSupport::SecurityUtils.secure_compare transformed_input, stored end end diff --git a/lib/gitlab/email/handler/create_note_handler.rb b/lib/gitlab/email/handler/create_note_handler.rb index b168efaac11..e6c64e2b1d6 100644 --- a/lib/gitlab/email/handler/create_note_handler.rb +++ b/lib/gitlab/email/handler/create_note_handler.rb @@ -50,7 +50,9 @@ module Gitlab end def create_note - sent_notification.create_reply(note_message) + external_author = from_address if author == User.support_bot + + sent_notification.create_reply(note_message, external_author) end def note_message diff --git a/lib/gitlab/email/hook/silent_mode_interceptor.rb b/lib/gitlab/email/hook/silent_mode_interceptor.rb new file mode 100644 index 00000000000..56f94119472 --- /dev/null +++ b/lib/gitlab/email/hook/silent_mode_interceptor.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module Hook + class SilentModeInterceptor + def self.delivering_email(message) + if Gitlab::CurrentSettings.silent_mode_enabled? + message.perform_deliveries = false + + Gitlab::AppJsonLogger.info( + message: "SilentModeInterceptor prevented sending mail", + mail_subject: message.subject, + silent_mode_enabled: true + ) + else + Gitlab::AppJsonLogger.debug( + message: "SilentModeInterceptor did nothing", + mail_subject: message.subject, + silent_mode_enabled: false + ) + end + end + end + end + end +end diff --git a/lib/gitlab/email/hook/validate_addresses_interceptor.rb b/lib/gitlab/email/hook/validate_addresses_interceptor.rb deleted file mode 100644 index e63f047e63d..00000000000 --- a/lib/gitlab/email/hook/validate_addresses_interceptor.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Email - module Hook - # Check for unsafe characters in the envelope-from and -to addresses. - # These are passed directly as arguments to sendmail and are liable to shell injection attacks: - # https://github.com/mikel/mail/blob/2.7.1/lib/mail/network/delivery_methods/sendmail.rb#L53-L58 - class ValidateAddressesInterceptor - UNSAFE_CHARACTERS = /(\\|[^[:print:]])/.freeze - - def self.delivering_email(message) - addresses = Array(message.smtp_envelope_from) + Array(message.smtp_envelope_to) - - addresses.each do |address| - next unless address.match?(UNSAFE_CHARACTERS) - - Gitlab::AuthLogger.info( - message: 'Skipping email with unsafe characters in address', - address: address, - subject: message.subject - ) - - message.perform_deliveries = false - - break - end - end - end - end - end -end diff --git a/lib/gitlab/email/html_parser.rb b/lib/gitlab/email/html_parser.rb index 10dbedbb464..693048adabf 100644 --- a/lib/gitlab/email/html_parser.rb +++ b/lib/gitlab/email/html_parser.rb @@ -34,11 +34,7 @@ module Gitlab end def filtered_text - @filtered_text ||= if Feature.enabled?(:service_desk_html_to_text_email_handler) - ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html) - else - Html2Text.convert(filtered_html) - end + @filtered_text ||= ::Gitlab::Email::HtmlToMarkdownParser.convert(filtered_html) end end end diff --git a/lib/gitlab/email/html_to_markdown_parser.rb b/lib/gitlab/email/html_to_markdown_parser.rb index 42dd012308b..5dd3725cc3e 100644 --- a/lib/gitlab/email/html_to_markdown_parser.rb +++ b/lib/gitlab/email/html_to_markdown_parser.rb @@ -5,25 +5,46 @@ require 'nokogiri' module Gitlab module Email class HtmlToMarkdownParser < Html2Text - ADDITIONAL_TAGS = %w[em strong img details].freeze - IMG_ATTRS = %w[alt src].freeze + extend Gitlab::Utils::Override + # List of tags to be converted by Markdown. + # + # All attributes are removed except for the defined ones. + # + # <tag> => [<attribute to keep>, ...] + ALLOWED_TAG_ATTRIBUTES = { + 'em' => [], + 'strong' => [], + 'details' => [], + 'img' => %w[alt src] + }.freeze + private_constant :ALLOWED_TAG_ATTRIBUTES + + # This redefinition can be removed once https://github.com/soundasleep/html2text_ruby/pull/30 + # is merged and released. def self.convert(html) html = fix_newlines(replace_entities(html)) doc = Nokogiri::HTML(html) - HtmlToMarkdownParser.new(doc).convert + new(doc).convert end + private + + override :iterate_over def iterate_over(node) - return super unless ADDITIONAL_TAGS.include?(node.name) + allowed_attributes = ALLOWED_TAG_ATTRIBUTES[node.name] + return super unless allowed_attributes - if node.name == 'img' - node.keys.each { |key| node.remove_attribute(key) unless IMG_ATTRS.include?(key) } # rubocop:disable Style/HashEachMethods - end + remove_attributes(node, allowed_attributes) Kramdown::Document.new(node.to_html, input: 'html').to_commonmark end + + def remove_attributes(node, allowed_attributes) + to_remove = (node.keys - allowed_attributes) + to_remove.each { |key| node.remove_attribute(key) } + end end end end diff --git a/lib/gitlab/email/incoming_email.rb b/lib/gitlab/email/incoming_email.rb new file mode 100644 index 00000000000..a0a01ae0d70 --- /dev/null +++ b/lib/gitlab/email/incoming_email.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module IncomingEmail + class << self + include Gitlab::Email::Common + + def config + incoming_email_config + end + + def key_from_address(address, wildcard_address: nil) + wildcard_address ||= config.address + regex = address_regex(wildcard_address) + return unless regex + + match = address.match(regex) + return unless match + + match[1] + end + + private + + def address_regex(wildcard_address) + return unless wildcard_address + + regex = Regexp.escape(wildcard_address) + regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') + Regexp.new(/\A<?#{regex}>?\z/).freeze + end + end + end + end +end diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 32794a6c99d..51d250ea98c 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -110,7 +110,7 @@ module Gitlab when String # Handle emails from clients which append with commas, # example clients are Microsoft exchange and iOS app - Gitlab::IncomingEmail.scan_fallback_references(references) + email_class.scan_fallback_references(references) when nil [] end @@ -177,7 +177,7 @@ module Gitlab def recipients_from_received_headers strong_memoize :emails_from_received_headers do - received.map { |header| header.value[RECEIVED_HEADER_REGEX, 1] }.compact + received.filter_map { |header| header.value[RECEIVED_HEADER_REGEX, 1] } end end @@ -203,7 +203,7 @@ module Gitlab end def email_class - Gitlab::IncomingEmail + Gitlab::Email::IncomingEmail end end end diff --git a/lib/gitlab/email/service_desk_email.rb b/lib/gitlab/email/service_desk_email.rb new file mode 100644 index 00000000000..4ea1c077327 --- /dev/null +++ b/lib/gitlab/email/service_desk_email.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Email + module ServiceDeskEmail + class << self + include Gitlab::Email::Common + + def config + Gitlab.config.service_desk_email + end + + def key_from_address(address) + wildcard_address = config&.address + return unless wildcard_address + + Gitlab::Email::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address) + end + + def address_for_key(key) + return if config.address.blank? + + config.address.sub(WILDCARD_PLACEHOLDER, key) + end + end + end + end +end diff --git a/lib/gitlab/email/service_desk_receiver.rb b/lib/gitlab/email/service_desk_receiver.rb index 6c6eb3b0a65..e286cf1f68c 100644 --- a/lib/gitlab/email/service_desk_receiver.rb +++ b/lib/gitlab/email/service_desk_receiver.rb @@ -12,7 +12,7 @@ module Gitlab end def email_class - ::Gitlab::ServiceDeskEmail + ::Gitlab::Email::ServiceDeskEmail end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 2b36b1c99bd..7d47bfe88fe 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -15,20 +15,6 @@ module Gitlab Rails.root.join("public/-/emojis/#{EMOJI_VERSION}") end - def emoji_image_tag(name, src) - image_options = { - class: 'emoji', - src: src, - title: ":#{name}:", - alt: ":#{name}:", - height: 20, - width: 20, - align: 'absmiddle' - } - - ActionController::Base.helpers.tag(:img, image_options) - end - # CSS sprite fallback takes precedence over image fallback # @param [TanukiEmoji::Character] emoji # @param [Hash] options diff --git a/lib/gitlab/encrypted_incoming_email_command.rb b/lib/gitlab/encrypted_incoming_email_command.rb index a18382439d6..05fc7cac000 100644 --- a/lib/gitlab/encrypted_incoming_email_command.rb +++ b/lib/gitlab/encrypted_incoming_email_command.rb @@ -8,7 +8,7 @@ module Gitlab class << self def encrypted_secrets - Gitlab::IncomingEmail.encrypted_secrets + Gitlab::Email::IncomingEmail.encrypted_secrets end def encrypted_file_template diff --git a/lib/gitlab/encrypted_service_desk_email_command.rb b/lib/gitlab/encrypted_service_desk_email_command.rb index ece6da7c1b3..1a0317e0da9 100644 --- a/lib/gitlab/encrypted_service_desk_email_command.rb +++ b/lib/gitlab/encrypted_service_desk_email_command.rb @@ -8,7 +8,7 @@ module Gitlab class << self def encrypted_secrets - Gitlab::ServiceDeskEmail.encrypted_secrets + Gitlab::Email::ServiceDeskEmail.encrypted_secrets end def encrypted_file_template diff --git a/lib/gitlab/etag_caching/router/graphql.rb b/lib/gitlab/etag_caching/router/graphql.rb index 7a0fb2ac269..92b21a0859d 100644 --- a/lib/gitlab/etag_caching/router/graphql.rb +++ b/lib/gitlab/etag_caching/router/graphql.rb @@ -16,12 +16,17 @@ module Gitlab [ %r(\Apipelines/sha/\w{7,40}\z), 'ci_editor', - 'pipeline_authoring' + 'pipeline_composition' ], [ %r(\Aon_demand_scan/counts/), 'on_demand_scans', 'dynamic_application_security_testing' + ], + [ + %r(\A/projects/.+/-/environments.json\z), + 'environment_details', + 'continuous_delivery' ] ].map { |attrs| build_graphql_route(*attrs) }.freeze diff --git a/lib/gitlab/event_store.rb b/lib/gitlab/event_store.rb index 023c8ace4d9..ce71ee594f2 100644 --- a/lib/gitlab/event_store.rb +++ b/lib/gitlab/event_store.rb @@ -60,6 +60,10 @@ module Gitlab store.subscribe ::MergeRequests::CreateApprovalNoteWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ResolveTodosAfterApprovalWorker, to: ::MergeRequests::ApprovedEvent store.subscribe ::MergeRequests::ExecuteApprovalHooksWorker, to: ::MergeRequests::ApprovedEvent + store.subscribe ::MergeRequests::SetReviewerReviewedWorker, to: ::MergeRequests::ApprovedEvent + store.subscribe ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker, + to: ::Packages::PackageCreatedEvent, + if: -> (event) { ::Ml::ExperimentTracking::AssociateMlCandidateToPackageWorker.handles_event?(event) } end private_class_method :configure! end diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index ce802b562f0..52ad67d6f8b 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -17,6 +17,10 @@ module Gitlab payload['exception.backtrace'] = Rails.backtrace_cleaner.clean(exception.backtrace) end + if exception.cause + payload['exception.cause_class'] = exception.cause.class.name + end + if sql = find_sql(exception) payload['exception.sql'] = sql end diff --git a/lib/gitlab/favicon.rb b/lib/gitlab/favicon.rb index 8e48b482462..f4633473a95 100644 --- a/lib/gitlab/favicon.rb +++ b/lib/gitlab/favicon.rb @@ -24,7 +24,7 @@ module Gitlab 'favicon-blue.png' end - def status_overlay(status_name) + def ci_status_overlay(status_name) path = File.join( 'ci_favicons', "#{status_name}.png" @@ -33,6 +33,15 @@ module Gitlab ActionController::Base.helpers.image_path(path, host: host) end + def mr_status_overlay(status_name) + path = File.join( + 'mr_favicons', + "#{status_name}.png" + ) + + ActionController::Base.helpers.image_path(path, host: host) + end + def available_status_names @available_status_names ||= Dir.glob(Rails.root.join('app', 'assets', 'images', 'ci_favicons', '*.png')) .map { |file| File.basename(file, '.png') } diff --git a/lib/gitlab/file_finder.rb b/lib/gitlab/file_finder.rb index 95f896a74e9..8a894901ca1 100644 --- a/lib/gitlab/file_finder.rb +++ b/lib/gitlab/file_finder.rb @@ -44,15 +44,11 @@ module Gitlab # Overridden in Gitlab::WikiFileFinder def search_paths(query) - if Feature.enabled?(:code_basic_search_files_by_regexp, project) - return [] if query.blank? || ref.blank? - - escaped_query = RE2::Regexp.escape(query) - query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") - repository.search_files_by_regexp(query_regexp, ref) - else - repository.search_files_by_name(query, ref) - end + return [] if query.blank? || ref.blank? + + escaped_query = RE2::Regexp.escape(query) + query_regexp = Gitlab::EncodingHelper.encode_utf8_no_detect("(?i)#{escaped_query}") + repository.search_files_by_regexp(query_regexp, ref) end end end diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 8e1b51fcec5..ef5c242e68a 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency 'gitlab/encoding_helper' +require_relative 'encoding_helper' module Gitlab module Git @@ -22,6 +22,21 @@ module Gitlab InvalidRefFormatError = Class.new(BaseError) ReferencesLockedError = Class.new(BaseError) + class ResourceExhaustedError < BaseError + def initialize(msg = nil, retry_after = 0) + super(msg) + @retry_after = retry_after + end + + def headers + if @retry_after.to_i > 0 + { "Retry-After" => @retry_after } + else + {} + end + end + end + class << self include Gitlab::EncodingHelper diff --git a/lib/gitlab/git/blame_mode.rb b/lib/gitlab/git/blame_mode.rb new file mode 100644 index 00000000000..15200e3888c --- /dev/null +++ b/lib/gitlab/git/blame_mode.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class BlameMode + def initialize(project, params) + @project = project + @params = params + end + + def streaming? + Gitlab::Utils.to_boolean(params[:streaming], default: false) + end + + def pagination? + return false if streaming? + return false if Gitlab::Utils.to_boolean(params[:no_pagination], default: false) + + Feature.enabled?(:blame_page_pagination, project) + end + + def full? + !streaming? && !pagination? + end + + private + + attr_reader :project, :params + end + end +end diff --git a/lib/gitlab/git/blame_pagination.rb b/lib/gitlab/git/blame_pagination.rb new file mode 100644 index 00000000000..6bf29859b14 --- /dev/null +++ b/lib/gitlab/git/blame_pagination.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module Git + class BlamePagination + include Gitlab::Utils::StrongMemoize + + PAGINATION_PER_PAGE = 1000 + STREAMING_FIRST_PAGE_SIZE = 200 + STREAMING_PER_PAGE = 2000 + + def initialize(blob, blame_mode, params) + @blob = blob + @blame_mode = blame_mode + @params = params + end + + def page + page = params.fetch(:page, 1).to_i + + return 1 if page < 1 + + page + end + strong_memoize_attr :page + + def per_page + blame_mode.streaming? ? STREAMING_PER_PAGE : PAGINATION_PER_PAGE + end + strong_memoize_attr :per_page + + def total_pages + total = (blob_lines_count.to_f / per_page).ceil + return total unless blame_mode.streaming? + + ([blob_lines_count - STREAMING_FIRST_PAGE_SIZE, 0].max.to_f / per_page).ceil + 1 + end + strong_memoize_attr :total_pages + + def total_extra_pages + [total_pages - 1, 0].max + end + strong_memoize_attr :total_extra_pages + + def paginator + return if blame_mode.streaming? || blame_mode.full? + + Kaminari.paginate_array([], total_count: blob_lines_count, limit: per_page) + .tap { |pagination| pagination.max_paginates_per(per_page) } + .page(page) + end + + def blame_range + return if blame_mode.full? + + first_line = ((page - 1) * per_page) + 1 + + if blame_mode.streaming? + return 1..STREAMING_FIRST_PAGE_SIZE if page == 1 + + first_line = STREAMING_FIRST_PAGE_SIZE + ((page - 2) * per_page) + 1 + end + + last_line = (first_line + per_page).to_i - 1 + + first_line..last_line + end + + private + + attr_reader :blob, :blame_mode, :params + + def blob_lines_count + @blob_lines_count ||= blob.data.lines.count + end + end + end +end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 267107e04e6..11eb0a584ab 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -16,7 +16,7 @@ module Gitlab SERIALIZE_KEYS = [ :id, :message, :parent_ids, :authored_date, :author_name, :author_email, - :committed_date, :committer_name, :committer_email, :trailers + :committed_date, :committer_name, :committer_email, :trailers, :referenced_by ].freeze attr_accessor(*SERIALIZE_KEYS) @@ -414,6 +414,7 @@ module Gitlab @committer_email = commit.committer.email.dup @parent_ids = Array(commit.parent_ids) @trailers = commit.trailers.to_h { |t| [t.key, t.value] } + @referenced_by = Array(commit.referenced_by) end # Gitaly provides a UNIX timestamp in author.date.seconds, and a timezone @@ -463,7 +464,8 @@ module Gitlab end def fetch_body_from_gitaly - self.class.get_message(@repository, id) + # #to_s is required to ensure BatchLoader is not returned. + self.class.get_message(@repository, id).to_s end def self.valid?(commit_id) diff --git a/lib/gitlab/git/diff_collection.rb b/lib/gitlab/git/diff_collection.rb index 0ffe8bee953..b4dd880ceb7 100644 --- a/lib/gitlab/git/diff_collection.rb +++ b/lib/gitlab/git/diff_collection.rb @@ -24,6 +24,8 @@ module Gitlab limits[:safe_max_lines] = [limits[:max_lines], defaults[:max_lines]].min limits[:safe_max_bytes] = limits[:safe_max_files] * 5.kilobytes # Average 5 KB per file limits[:max_patch_bytes] = Gitlab::Git::Diff.patch_hard_limit_bytes + limits[:max_patch_bytes_for_file_extension] = options.fetch(:max_patch_bytes_for_file_extension, {}) + limits end diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 1d7966a11ed..4a09f866db4 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -36,8 +36,6 @@ module Gitlab target.name elsif target.is_a? String target - else - nil end end end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index e1399b6642b..80d0fd17568 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -54,8 +54,6 @@ module Gitlab # state. alias_method :object_pool_remote_name, :gl_repository - # This initializer method is only used on the client side (gitlab-ce). - # Gitaly-ruby uses a different initializer. def initialize(storage, relative_path, gl_repository, gl_project_path, container: nil) @storage = storage @relative_path = relative_path @@ -263,8 +261,12 @@ module Gitlab def archive_metadata(ref, storage_path, project_path, format = "tar.gz", append_sha:, path: nil) ref ||= root_ref - commit_id = extract_commit_id_from_ref(ref) - return {} if commit_id.nil? + if Feature.enabled?(:resolve_ambiguous_archives, container) + commit_id = extract_commit_id_from_ref(ref) + return {} if commit_id.nil? + else + commit_id = ref + end commit = Gitlab::Git::Commit.find(self, commit_id) return {} if commit.nil? @@ -807,27 +809,17 @@ module Gitlab end end - def license(from_gitaly) + def license wrapped_gitaly_errors do response = gitaly_repository_client.find_license break nil if response.license_short_name.empty? - if from_gitaly - break ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, - name: response.license_name, - nickname: response.license_nickname.presence, - url: response.license_url.presence, - path: response.license_path) - end - - licensee_object = Licensee::License.new(response.license_short_name) - - break nil if licensee_object.name.blank? - - licensee_object.meta.nickname = "LICENSE" if licensee_object.key == "other" - - licensee_object + ::Gitlab::Git::DeclaredLicense.new(key: response.license_short_name, + name: response.license_name, + nickname: response.license_nickname.presence, + url: response.license_url.presence, + path: response.license_path) end rescue Licensee::InvalidLicense => e Gitlab::ErrorTracking.track_exception(e) @@ -1152,7 +1144,7 @@ module Gitlab def checksum # The exists? RPC is much cheaper, so we perform this request first - raise NoRepository, "Repository does not exists" unless exists? + raise NoRepository, "Repository does not exist" unless exists? gitaly_repository_client.calculate_checksum rescue GRPC::NotFound diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb index 66cfc02130b..c7a981c7dd4 100644 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ b/lib/gitlab/git/rugged_impl/tree.rb @@ -130,7 +130,6 @@ module Gitlab new( id: entry[:oid], - root_id: root_tree.oid, name: entry[:name], type: entry[:type], mode: entry[:filemode].to_s(8), diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 5ed5158eeea..37977a1dfb6 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -96,8 +96,6 @@ module Gitlab nil # not implemented, see https://gitlab.com/gitlab-org/gitlab/issues/19260 when :X509 X509::Tag.new(@repository, self).signature - else - nil end end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index f0eef619e13..e437f99dab3 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::EncodingHelper extend Gitlab::Git::WrapsGitalyErrors - attr_accessor :id, :root_id, :type, :mode, :commit_id, :submodule_url + attr_accessor :id, :type, :mode, :commit_id, :submodule_url attr_writer :name, :path, :flat_path class << self @@ -61,7 +61,7 @@ module Gitlab end def initialize(options) - %w(id root_id name path flat_path type mode commit_id).each do |key| + %w(id name path flat_path type mode commit_id).each do |key| self.send("#{key}=", options[key.to_sym]) # rubocop:disable GitlabSecurity/PublicSend end end diff --git a/lib/gitlab/git/wraps_gitaly_errors.rb b/lib/gitlab/git/wraps_gitaly_errors.rb index 1d34f3c8eb2..20bcf3585e1 100644 --- a/lib/gitlab/git/wraps_gitaly_errors.rb +++ b/lib/gitlab/git/wraps_gitaly_errors.rb @@ -5,14 +5,40 @@ module Gitlab module WrapsGitalyErrors def wrapped_gitaly_errors(&block) yield block - rescue GRPC::NotFound => e - raise Gitlab::Git::Repository::NoRepository, e - rescue GRPC::InvalidArgument => e - raise ArgumentError, e - rescue GRPC::DeadlineExceeded => e - raise Gitlab::Git::CommandTimedOut, e rescue GRPC::BadStatus => e - raise Gitlab::Git::CommandError, e + # The GRPC::BadStatus is the fundamental error that serves as the basis for all other gRPC error categories, + # including GRPC::InvalidArgument. It is essential to note that rescuing the specific exception class does not + # account for all possible cases. In this regard, a status exception can be directly generated from + # GRPC::BadStatus. Therefore, it is recommended that we capture and rescue the GRPC::BadStatus and assert the + # status code to ensure adequate coverage of error cases. + case e.code + when GRPC::Core::StatusCodes::NOT_FOUND + raise Gitlab::Git::Repository::NoRepository, e + when GRPC::Core::StatusCodes::INVALID_ARGUMENT + raise ArgumentError, e + when GRPC::Core::StatusCodes::DEADLINE_EXCEEDED + raise Gitlab::Git::CommandTimedOut, e + when GRPC::Core::StatusCodes::RESOURCE_EXHAUSTED + handle_resource_exhausted(e) + else + raise Gitlab::Git::CommandError, e + end + end + + private + + def handle_resource_exhausted(exception) + detail = Gitlab::GitalyClient.decode_detailed_error(exception) + + case detail.class.name + when Gitaly::LimitError.name + retry_after = detail&.retry_after&.seconds + raise ResourceExhaustedError.new( + "Upstream Gitaly has been exhausted: #{detail.error_message}. Try again later", retry_after + ) + else + raise ResourceExhaustedError, _("Upstream Gitaly has been exhausted. Try again later") + end end end end diff --git a/lib/gitlab/git_access_design.rb b/lib/gitlab/git_access_design.rb index bf89c01305a..fb8df0d217a 100644 --- a/lib/gitlab/git_access_design.rb +++ b/lib/gitlab/git_access_design.rb @@ -4,6 +4,23 @@ module Gitlab class GitAccessDesign < GitAccess extend ::Gitlab::Utils::Override + # TODO Re-factor so that correct container is passed to the constructor + # and this method can be removed from here + # https://gitlab.com/gitlab-org/gitlab/-/issues/409454 + def initialize( + actor, container, protocol, authentication_abilities:, repository_path: nil, redirected_path: nil, + auth_result_type: nil) + super( + actor, + select_container(container), + protocol, + authentication_abilities: authentication_abilities, + repository_path: repository_path, + redirected_path: redirected_path, + auth_result_type: auth_result_type + ) + end + def check(_cmd, _changes) check_protocol! check_can_create_design! @@ -18,6 +35,10 @@ module Gitlab private + def select_container(container) + container.is_a?(::DesignManagement::Repository) ? container.project : container + end + def check_protocol! if protocol != 'web' raise ::Gitlab::GitAccess::ForbiddenError, "Designs are only accessible using the web interface" diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index f4d4cebc096..7867e1b8c37 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -12,10 +12,10 @@ module Gitlab # Validates a given name against the git reference specification # # Returns true for a valid reference name, false otherwise - def validate(ref_name) + def validate(ref_name, skip_head_ref_check: false) return false if ref_name.to_s.empty? # #blank? raises an ArgumentError for invalid encodings return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) - return false if ref_name == 'HEAD' + return false if ref_name == 'HEAD' && !skip_head_ref_check begin Rugged::Reference.valid_name?("refs/heads/#{ref_name}") diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 4df9d800ea6..0c67b9fa078 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -146,7 +146,6 @@ module Gitlab message.entries.map do |gitaly_tree_entry| Gitlab::Git::Tree.new( id: gitaly_tree_entry.oid, - root_id: gitaly_tree_entry.root_oid, type: gitaly_tree_entry.type.downcase, mode: gitaly_tree_entry.mode.to_s(8), name: File.basename(gitaly_tree_entry.path), @@ -423,7 +422,8 @@ module Gitlab first_parent: !!options[:first_parent], global_options: parse_global_options!(options), disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. - trailers: options[:trailers] + trailers: options[:trailers], + include_referenced_by: options[:include_referenced_by] ) request.after = GitalyClient.timestamp(options[:after]) if options[:after] request.before = GitalyClient.timestamp(options[:before]) if options[:before] @@ -441,14 +441,14 @@ module Gitlab # revision exists, or `false` otherwise. This function accepts all revisions as specified by # gitrevisions(1). def object_existence_map(revisions, gitaly_repo: @gitaly_repo) + return {} unless revisions.present? + enum = Enumerator.new do |y| - # This is a bug in Gitaly: revisions of the initial request are ignored. This will be fixed in v15.0 via - # https://gitlab.com/gitlab-org/gitaly/-/merge_requests/4510, so we can merge initial request and the initial - # set of revisions starting with v15.1. - y.yield Gitaly::CheckObjectsExistRequest.new(repository: gitaly_repo) + revisions.each_slice(100).with_index do |revisions_subset, i| + params = { revisions: revisions_subset } + params[:repository] = gitaly_repo if i == 0 - revisions.each_slice(100) do |revisions_subset| - y.yield Gitaly::CheckObjectsExistRequest.new(revisions: revisions_subset) + y.yield Gitaly::CheckObjectsExistRequest.new(**params) end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 313334737c0..1af06cc7490 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -45,7 +45,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :access_check access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls @@ -82,7 +82,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), fallback_message: e.details) @@ -124,7 +124,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :custom_hook raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), fallback_message: e.details) @@ -188,7 +188,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :access_check access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls @@ -247,7 +247,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :access_check access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls @@ -329,7 +329,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :access_check access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls @@ -366,7 +366,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :resolve_revision, :rebase_conflict # Theoretically, we could now raise specific errors based on the type # of the detailed error. Most importantly, we get error details when @@ -458,7 +458,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :access_check access_check_error = detailed_error.access_check # These messages were returned from internal/allowed API calls diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index ac6491e8770..88c79eb8954 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -113,7 +113,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :tag_not_found raise Gitlab::Git::UnknownRef, "tag does not exist: #{tag_name}" else @@ -135,7 +135,7 @@ module Gitlab rescue GRPC::BadStatus => e detailed_error = GitalyClient.decode_detailed_error(e) - case detailed_error&.error + case detailed_error.try(:error) when :invalid_format raise Gitlab::Git::InvalidRefFormatError, "references have an invalid format: #{detailed_error.invalid_format.refs.join(",")}" when :references_locked @@ -239,7 +239,7 @@ module Gitlab sort_by = 'name' if sort_by == 'name_asc' enum_value = Gitaly::FindLocalBranchesRequest::SortBy.resolve(sort_by.upcase.to_sym) - raise ArgumentError, "Invalid sort_by key `#{sort_by}`" unless enum_value + return Gitaly::FindLocalBranchesRequest::SortBy::NAME unless enum_value enum_value end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index bcc03ca08c9..93d58710b0c 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -109,7 +109,7 @@ module Gitlab # rubocop: enable Metrics/ParameterLists def create_repository(default_branch = nil) - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch)) gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end @@ -306,18 +306,18 @@ module Gitlab end def search_files_by_name(ref, query, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query, options = {}) - request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) + request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: query) response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response, options) end def search_files_by_regexp(ref, filter, limit: 0, offset: 0) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: encode_binary(ref), query: '.', filter: filter, limit: limit, offset: offset) gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end diff --git a/lib/gitlab/github_import/bulk_importing.rb b/lib/gitlab/github_import/bulk_importing.rb index 0c91eff1d10..d16f4d7587b 100644 --- a/lib/gitlab/github_import/bulk_importing.rb +++ b/lib/gitlab/github_import/bulk_importing.rb @@ -27,8 +27,13 @@ module Gitlab build_record = model.new(attrs) if build_record.invalid? - log_error(object[:id], build_record.errors.full_messages) - errors << build_record.errors + github_identifiers = github_identifiers(object) + + log_error(github_identifiers, build_record.errors.full_messages) + errors << { + validation_errors: build_record.errors, + github_identifiers: github_identifiers + } next end @@ -53,17 +58,18 @@ module Gitlab raise NotImplementedError end - def bulk_insert_failures(validation_errors) - rows = validation_errors.map do |error| + def bulk_insert_failures(errors) + rows = errors.map do |error| correlation_id_value = Labkit::Correlation::CorrelationId.current_or_new_id { source: self.class.name, exception_class: 'ActiveRecord::RecordInvalid', - exception_message: error.full_messages.first.truncate(255), + exception_message: error[:validation_errors].full_messages.first.truncate(255), correlation_id_value: correlation_id_value, retry_count: nil, - created_at: Time.zone.now + created_at: Time.zone.now, + external_identifiers: error[:github_identifiers] } end @@ -88,15 +94,19 @@ module Gitlab ) end - def log_error(object_id, messages) + def log_error(github_identifiers, messages) Gitlab::Import::Logger.error( import_type: :github, project_id: project.id, importer: self.class.name, message: messages, - github_identifier: object_id + github_identifiers: github_identifiers ) end + + def github_identifiers(object) + raise NotImplementedError + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 1c9ca9f43a8..886563a6f69 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -112,6 +112,10 @@ module Gitlab each_object(:branches, *args) end + def collaborators(*args) + each_object(:collaborators, *args) + end + def branch_protection(repo_name, branch_name) with_rate_limit { octokit.branch_protection(repo_name, branch_name).to_h } end diff --git a/lib/gitlab/github_import/clients/proxy.rb b/lib/gitlab/github_import/clients/proxy.rb index b12df404640..27030f5382a 100644 --- a/lib/gitlab/github_import/clients/proxy.rb +++ b/lib/gitlab/github_import/clients/proxy.rb @@ -6,6 +6,10 @@ module Gitlab class Proxy attr_reader :client + delegate :each_object, :user, :octokit, to: :client + + REPOS_COUNT_CACHE_KEY = 'github-importer/provider-repo-count/%{type}/%{user_id}' + def initialize(access_token, client_options) @client = pick_client(access_token, client_options) end @@ -13,24 +17,26 @@ module Gitlab def repos(search_text, options) return { repos: filtered(client.repos, search_text) } if use_legacy? - if use_graphql? - fetch_repos_via_graphql(search_text, options) - else - fetch_repos_via_rest(search_text, options) - end + fetch_repos_via_graphql(search_text, options) end - private + def count_repos_by(relation_type, user_id) + return if use_legacy? + + key = format(REPOS_COUNT_CACHE_KEY, type: relation_type, user_id: user_id) - def fetch_repos_via_rest(search_text, options) - { repos: client.search_repos_by_name(search_text, options)[:items] } + ::Gitlab::Cache::Import::Caching.read_integer(key, timeout: 5.minutes) || + fetch_and_cache_repos_count_via_graphql(relation_type, key) end + private + def fetch_repos_via_graphql(search_text, options) response = client.search_repos_by_name_graphql(search_text, options) { repos: response.dig(:data, :search, :nodes), - page_info: response.dig(:data, :search, :pageInfo) + page_info: response.dig(:data, :search, :pageInfo), + count: response.dig(:data, :search, :repositoryCount) } end @@ -50,8 +56,11 @@ module Gitlab Feature.disabled?(:remove_legacy_github_client) end - def use_graphql? - Feature.enabled?(:github_client_fetch_repos_via_graphql) + def fetch_and_cache_repos_count_via_graphql(relation_type, key) + response = client.count_repos_by_relation_type_graphql(relation_type: relation_type) + count = response.dig(:data, :search, :repositoryCount) + + ::Gitlab::Cache::Import::Caching.write(key, count, timeout: 5.minutes) end end end diff --git a/lib/gitlab/github_import/clients/search_repos.rb b/lib/gitlab/github_import/clients/search_repos.rb index b72e5ac7751..a2ef6ca24eb 100644 --- a/lib/gitlab/github_import/clients/search_repos.rb +++ b/lib/gitlab/github_import/clients/search_repos.rb @@ -5,24 +5,24 @@ module Gitlab module Clients module SearchRepos def search_repos_by_name_graphql(name, options = {}) - with_retry do - octokit.post( - '/graphql', - { query: graphql_search_repos_body(name, options) }.to_json - ).to_h - end + graphql_request(graphql_search_repos_body(name, options)) + end + + def count_repos_by_relation_type_graphql(options) + graphql_request(count_by_relation_type_query(options)) end - def search_repos_by_name(name, options = {}) - search_query = search_repos_query(name, options) + private + def graphql_request(query) with_retry do - octokit.search_repositories(search_query, options).to_h + octokit.post( + '/graphql', + { query: query }.to_json + ).to_h end end - private - def graphql_search_repos_body(name, options) query = search_repos_query(name, options) query = "query: \"#{query}\"" @@ -45,14 +45,15 @@ module Gitlab endCursor hasNextPage hasPreviousPage - } + }, + repositoryCount } } TEXT end def search_repos_query(string, options = {}) - base = "#{string} in:name is:public,private" + base = "#{string} in:name is:public,private fork:true" case options[:relation_type] when 'organization' then organization_repos_query(base, options) @@ -64,7 +65,11 @@ module Gitlab end def organization_repos_query(search_string, options) - "#{search_string} org:#{options[:organization_login]}" + if options[:organization_login].present? + "#{search_string} org:#{options[:organization_login]}" + else + organizations_subquery + end end def collaborated_repos_query(search_string) @@ -95,6 +100,18 @@ module Gitlab .map { |org| "org:#{org[:login]}" } .join(' ') end + + def count_by_relation_type_query(options) + query = search_repos_query(nil, options) + query = "query: \"#{query}\"" + <<-TEXT + { + search(type: REPOSITORY, #{query}) { + repositoryCount + } + } + TEXT + end end end end diff --git a/lib/gitlab/github_import/importer/attachments/issues_importer.rb b/lib/gitlab/github_import/importer/attachments/issues_importer.rb index 090bfb4a098..c8f0b59fd18 100644 --- a/lib/gitlab/github_import/importer/attachments/issues_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/issues_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.issues.select(:id, :description) + project.issues.select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb index f41071b1785..cd3a327a846 100644 --- a/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/merge_requests_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.merge_requests.select(:id, :description) + project.merge_requests.select(:id, :description, :iid) end def ordering_column diff --git a/lib/gitlab/github_import/importer/attachments/releases_importer.rb b/lib/gitlab/github_import/importer/attachments/releases_importer.rb index feaa69eff71..7d6dbeb901e 100644 --- a/lib/gitlab/github_import/importer/attachments/releases_importer.rb +++ b/lib/gitlab/github_import/importer/attachments/releases_importer.rb @@ -24,7 +24,7 @@ module Gitlab private def collection - project.releases.select(:id, :description) + project.releases.select(:id, :description, :tag) end end end diff --git a/lib/gitlab/github_import/importer/collaborator_importer.rb b/lib/gitlab/github_import/importer/collaborator_importer.rb new file mode 100644 index 00000000000..9a90ea5a4ed --- /dev/null +++ b/lib/gitlab/github_import/importer/collaborator_importer.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class CollaboratorImporter + attr_reader :collaborator, :project, :client, :members_finder + + # collaborator - An instance of `Gitlab::GithubImport::Representation::Collaborator` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(collaborator, project, client) + @collaborator = collaborator + @project = project + @client = client + @members_finder = ::MembersFinder.new(project, project.creator) + end + + def execute + user_finder = GithubImport::UserFinder.new(project, client) + user_id = user_finder.user_id_for(collaborator) + return if user_id.nil? + + membership = existing_user_membership(user_id) + access_level = map_access_level + return if membership && membership[:access_level] >= map_access_level + + create_membership!(user_id, access_level) + end + + private + + def existing_user_membership(user_id) + members_finder.execute.find_by_user_id(user_id) + end + + def map_access_level + access_level = + case collaborator[:role_name] + when 'read' then Gitlab::Access::GUEST + when 'triage' then Gitlab::Access::REPORTER + when 'write' then Gitlab::Access::DEVELOPER + when 'maintain' then Gitlab::Access::MAINTAINER + when 'admin' then Gitlab::Access::OWNER + end + return access_level if access_level + + raise( + ::Gitlab::GithubImport::ObjectImporter::NotRetriableError, + "Unknown GitHub role: #{collaborator[:role_name]}" + ) + end + + def create_membership!(user_id, access_level) + ::ProjectMember.create!( + source: project, + access_level: access_level, + user_id: user_id, + member_namespace_id: project.project_namespace_id, + created_by_id: project.creator_id + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/collaborators_importer.rb b/lib/gitlab/github_import/importer/collaborators_importer.rb new file mode 100644 index 00000000000..7b18d3dba2a --- /dev/null +++ b/lib/gitlab/github_import/importer/collaborators_importer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class CollaboratorsImporter + include ParallelScheduling + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + direct_collaborators = client.collaborators(repo, affiliation: 'direct') + outside_collaborators = client.collaborators(repo, affiliation: 'outside') + collaborators_to_import = direct_collaborators.to_a - outside_collaborators.to_a + + collaborators_to_import.each do |collaborator| + next if already_imported?(collaborator) + + yield collaborator + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + mark_as_imported(collaborator) + end + end + + def importer_class + CollaboratorImporter + end + + def representation_class + Representation::Collaborator + end + + def sidekiq_worker_class + ImportCollaboratorWorker + end + + def object_type + :collaborator + end + + def collection_method + :collaborators + end + + def id_for_already_imported_cache(collaborator) + collaborator[:id] + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index b56ae186d3c..4fe371e5900 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -55,6 +55,7 @@ module Gitlab record = record_class.new(id: db_id, iid: iid) record.project = project + record.namespace = project.project_namespace if record.respond_to?(:namespace) record.readonly! record end diff --git a/lib/gitlab/github_import/importer/label_links_importer.rb b/lib/gitlab/github_import/importer/label_links_importer.rb index 52c87dda347..a20fec4b2ba 100644 --- a/lib/gitlab/github_import/importer/label_links_importer.rb +++ b/lib/gitlab/github_import/importer/label_links_importer.rb @@ -25,6 +25,8 @@ module Gitlab items = [] target_id = find_target_id + return if target_id.blank? + issue.label_names.each do |label_name| # Although unlikely it's technically possible for an issue to be # given a label that was created and assigned after we imported all diff --git a/lib/gitlab/github_import/importer/labels_importer.rb b/lib/gitlab/github_import/importer/labels_importer.rb index d5d1cd28b7c..4554b932520 100644 --- a/lib/gitlab/github_import/importer/labels_importer.rb +++ b/lib/gitlab/github_import/importer/labels_importer.rb @@ -53,9 +53,18 @@ module Gitlab :label end + private + def model Label end + + def github_identifiers(label) + { + title: label[:name], + object_type: object_type + } + end end end end diff --git a/lib/gitlab/github_import/importer/milestones_importer.rb b/lib/gitlab/github_import/importer/milestones_importer.rb index 560fbdc66e3..cd6d450f15b 100644 --- a/lib/gitlab/github_import/importer/milestones_importer.rb +++ b/lib/gitlab/github_import/importer/milestones_importer.rb @@ -57,9 +57,19 @@ module Gitlab :milestone end + private + def model Milestone end + + def github_identifiers(milestone) + { + iid: milestone[:number], + title: milestone[:title], + object_type: object_type + } + end end end end diff --git a/lib/gitlab/github_import/importer/note_attachments_importer.rb b/lib/gitlab/github_import/importer/note_attachments_importer.rb index 9901c9e76f5..266ee2938ba 100644 --- a/lib/gitlab/github_import/importer/note_attachments_importer.rb +++ b/lib/gitlab/github_import/importer/note_attachments_importer.rb @@ -6,7 +6,7 @@ module Gitlab class NoteAttachmentsImporter attr_reader :note_text, :project - # note_text - An instance of `NoteText`. + # note_text - An instance of `Gitlab::GithubImport::Representation::NoteText`. # project - An instance of `Project`. # client - An instance of `Gitlab::GithubImport::Client`. def initialize(note_text, project, _client = nil) @@ -19,7 +19,7 @@ module Gitlab return if attachments.blank? new_text = attachments.reduce(note_text.text) do |text, attachment| - new_url = download_attachment(attachment) + new_url = gitlab_attachment_link(attachment) text.gsub(attachment.url, new_url) end @@ -28,6 +28,28 @@ module Gitlab private + def gitlab_attachment_link(attachment) + project_import_source = project.import_source + + if attachment.part_of_project_blob?(project_import_source) + convert_project_content_link(attachment.url, project_import_source) + elsif attachment.media? || attachment.doc_belongs_to_project?(project_import_source) + download_attachment(attachment) + else # url to other GitHub project + attachment.url + end + end + + # From: https://github.com/login/test-import-attachments-source/blob/main/example.md + # To: https://gitlab.com/login/test-import-attachments-target/-/blob/main/example.md + def convert_project_content_link(attachment_url, import_source) + path_without_domain = attachment_url.gsub(::Gitlab::GithubImport::MarkdownText.github_url, '') + path_without_import_source = path_without_domain.gsub(import_source, '').delete_prefix('/') + path_with_blob_prefix = "/-#{path_without_import_source}" + + ::Gitlab::Routing.url_helpers.project_url(project) + path_with_blob_prefix + end + # in: an instance of Gitlab::GithubImport::Markdown::Attachment # out: gitlab attachment markdown url def download_attachment(attachment) diff --git a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb deleted file mode 100644 index f05aa26a449..00000000000 --- a/lib/gitlab/github_import/importer/pull_request_merged_by_importer.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class PullRequestMergedByImporter - # pull_request - An instance of - # `Gitlab::GithubImport::Representation::PullRequest` - # project - An instance of `Project` - # client - An instance of `Gitlab::GithubImport::Client` - def initialize(pull_request, project, client) - @pull_request = pull_request - @project = project - @client = client - end - - def execute - user_finder = GithubImport::UserFinder.new(project, client) - - gitlab_user_id = begin - user_finder.user_id_for(pull_request.merged_by) - rescue ::Octokit::NotFound - nil - end - - metrics_upsert(gitlab_user_id) - - add_note! - end - - private - - attr_reader :project, :pull_request, :client - - def metrics_upsert(gitlab_user_id) - MergeRequest::Metrics.upsert({ - target_project_id: project.id, - merge_request_id: merge_request.id, - merged_by_id: gitlab_user_id, - merged_at: pull_request.merged_at, - created_at: timestamp, - updated_at: timestamp - }, unique_by: :merge_request_id) - end - - def add_note! - merge_request.notes.create!( - importing: true, - note: missing_author_note, - author_id: project.creator_id, - project: project, - created_at: pull_request.merged_at - ) - end - - def merge_request - @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid) - end - - def timestamp - @timestamp ||= Time.new.utc - end - - def missing_author_note - s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*") % { - author: pull_request.merged_by&.login || 'ghost', - timestamp: pull_request.merged_at - } - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb deleted file mode 100644 index b1e259fe940..00000000000 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ /dev/null @@ -1,145 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class PullRequestReviewImporter - # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` - # project - An instance of `Project` - # client - An instance of `Gitlab::GithubImport::Client` - def initialize(review, project, client) - @review = review - @project = project - @client = client - @merge_request = project.merge_requests.find_by_id(review.merge_request_id) - end - - def execute - user_finder = GithubImport::UserFinder.new(project, client) - - gitlab_user_id = begin - user_finder.user_id_for(review.author) - rescue ::Octokit::NotFound - nil - end - - if gitlab_user_id - add_review_note!(gitlab_user_id) - add_approval!(gitlab_user_id) - add_reviewer!(gitlab_user_id) - else - add_complementary_review_note!(project.creator_id) - end - end - - private - - attr_reader :review, :merge_request, :project, :client - - def add_review_note!(author_id) - return if review.note.empty? - - add_note!(author_id, review_note_content) - end - - def add_complementary_review_note!(author_id) - return if review.note.empty? && !review.approval? - - note_body = MarkdownText.format( - review_note_content, - review.author - ) - - add_note!(author_id, note_body) - end - - def review_note_content - header = "**Review:** #{review.review_type.humanize}" - - if review.note.present? - "#{header}\n\n#{review.note}" - else - header - end - end - - def add_note!(author_id, note) - note = Note.new(note_attributes(author_id, note)) - - note.save! - end - - def note_attributes(author_id, note, extra = {}) - { - importing: true, - noteable_id: merge_request.id, - noteable_type: 'MergeRequest', - project_id: project.id, - author_id: author_id, - note: note, - system: false, - created_at: submitted_at, - updated_at: submitted_at - }.merge(extra) - end - - def add_approval!(user_id) - return unless review.review_type == 'APPROVED' - - approval_attribues = { - merge_request_id: merge_request.id, - user_id: user_id, - created_at: submitted_at, - updated_at: submitted_at - } - - result = ::Approval.insert( - approval_attribues, - returning: [:id], - unique_by: [:user_id, :merge_request_id] - ) - - if result.rows.present? - add_approval_system_note!(user_id) - end - end - - def add_reviewer!(user_id) - return if review_re_requested?(user_id) - - ::MergeRequestReviewer.create!( - merge_request_id: merge_request.id, - user_id: user_id, - state: ::MergeRequestReviewer.states['reviewed'], - created_at: submitted_at - ) - rescue ActiveRecord::RecordNotUnique - # multiple reviews from single person could make a SQL concurrency issue here - nil - end - - # rubocop:disable CodeReuse/ActiveRecord - def review_re_requested?(user_id) - # records that were imported on previous stage with "unreviewed" status - MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? - end - # rubocop:enable CodeReuse/ActiveRecord - - def add_approval_system_note!(user_id) - attributes = note_attributes( - user_id, - 'approved this merge request', - system: true, - system_note_metadata: SystemNoteMetadata.new(action: 'approved') - ) - - Note.create!(attributes) - end - - def submitted_at - @submitted_at ||= (review.submitted_at || merge_request.updated_at) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb new file mode 100644 index 00000000000..9aa55fd3eae --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/all_merged_by_importer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class AllMergedByImporter + include ParallelScheduling + + def importer_class + MergedByImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequest + end + + def sidekiq_worker_class + Gitlab::GithubImport::PullRequests::ImportMergedByWorker + end + + def collection_method + :pull_requests_merged_by + end + + def object_type + :pull_request_merged_by + end + + def id_for_already_imported_cache(merge_request) + merge_request.id + end + + def each_object_to_import + merge_requests_to_import.find_each do |merge_request| + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + pull_request = client.pull_request(project.import_source, merge_request.iid) + yield(pull_request) + + mark_as_imported(merge_request) + end + end + + private + + # Returns only the merge requests that still have merged_by to be imported. + def merge_requests_to_import + project.merge_requests.id_not_in(already_imported_objects).with_state(:merged) + end + + def already_imported_objects + Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb new file mode 100644 index 00000000000..19880716832 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/merged_by_importer.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class MergedByImporter + # pull_request - An instance of + # `Gitlab::GithubImport::Representation::PullRequest` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(pull_request, project, client) + @pull_request = pull_request + @project = project + @client = client + end + + def execute + user_finder = GithubImport::UserFinder.new(project, client) + + gitlab_user_id = user_finder.user_id_for(pull_request.merged_by) + + metrics_upsert(gitlab_user_id) + + add_note! + end + + private + + attr_reader :project, :pull_request, :client + + def metrics_upsert(gitlab_user_id) + MergeRequest::Metrics.upsert({ + target_project_id: project.id, + merge_request_id: merge_request.id, + merged_by_id: gitlab_user_id, + merged_at: pull_request.merged_at, + created_at: timestamp, + updated_at: timestamp + }, unique_by: :merge_request_id) + end + + def add_note! + merge_request.notes.create!( + importing: true, + note: missing_author_note, + author_id: project.creator_id, + project: project, + created_at: pull_request.merged_at + ) + end + + def merge_request + @merge_request ||= project.merge_requests.find_by_iid(pull_request.iid) + end + + def timestamp + @timestamp ||= Time.new.utc + end + + def missing_author_note + format(s_("GitHubImporter|*Merged by: %{author} at %{timestamp}*"), + author: pull_request.merged_by&.login || 'ghost', + timestamp: pull_request.merged_at + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb new file mode 100644 index 00000000000..b250a42a53c --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/review_importer.rb @@ -0,0 +1,141 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewImporter + # review - An instance of `Gitlab::GithubImport::Representation::PullRequestReview` + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(review, project, client) + @review = review + @project = project + @client = client + @merge_request = project.merge_requests.find_by_id(review.merge_request_id) + end + + def execute + user_finder = GithubImport::UserFinder.new(project, client) + + gitlab_user_id = user_finder.user_id_for(review.author) + + if gitlab_user_id + add_review_note!(gitlab_user_id) + add_approval!(gitlab_user_id) + add_reviewer!(gitlab_user_id) + else + add_complementary_review_note!(project.creator_id) + end + end + + private + + attr_reader :review, :merge_request, :project, :client + + def add_review_note!(author_id) + return if review.note.empty? + + add_note!(author_id, review_note_content) + end + + def add_complementary_review_note!(author_id) + return if review.note.empty? && !review.approval? + + note_body = MarkdownText.format( + review_note_content, + review.author + ) + + add_note!(author_id, note_body) + end + + def review_note_content + header = "**Review:** #{review.review_type.humanize}" + + if review.note.present? + "#{header}\n\n#{review.note}" + else + header + end + end + + def add_note!(author_id, note) + note = Note.new(note_attributes(author_id, note)) + + note.save! + end + + def note_attributes(author_id, note, extra = {}) + { + importing: true, + noteable_id: merge_request.id, + noteable_type: 'MergeRequest', + project_id: project.id, + author_id: author_id, + note: note, + system: false, + created_at: submitted_at, + updated_at: submitted_at + }.merge(extra) + end + + def add_approval!(user_id) + return unless review.review_type == 'APPROVED' + + approval_attribues = { + merge_request_id: merge_request.id, + user_id: user_id, + created_at: submitted_at, + updated_at: submitted_at + } + + result = ::Approval.insert( + approval_attribues, + returning: [:id], + unique_by: [:user_id, :merge_request_id] + ) + + add_approval_system_note!(user_id) if result.rows.present? + end + + def add_reviewer!(user_id) + return if review_re_requested?(user_id) + + ::MergeRequestReviewer.create!( + merge_request_id: merge_request.id, + user_id: user_id, + state: ::MergeRequestReviewer.states['reviewed'], + created_at: submitted_at + ) + rescue ActiveRecord::RecordNotUnique + # multiple reviews from single person could make a SQL concurrency issue here + nil + end + + # rubocop:disable CodeReuse/ActiveRecord + def review_re_requested?(user_id) + # records that were imported on previous stage with "unreviewed" status + MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? + end + # rubocop:enable CodeReuse/ActiveRecord + + def add_approval_system_note!(user_id) + attributes = note_attributes( + user_id, + 'approved this merge request', + system: true, + system_note_metadata: SystemNoteMetadata.new(action: 'approved') + ) + + Note.create!(attributes) + end + + def submitted_at + @submitted_at ||= (review.submitted_at || merge_request.updated_at) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb index bb51d856d9b..f51c610f24b 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb @@ -8,7 +8,6 @@ module Gitlab def initialize(review_request, project, client) @review_request = review_request @user_finder = UserFinder.new(project, client) - @issue_finder = IssuableFinder.new(project, client) end def execute @@ -20,7 +19,7 @@ module Gitlab attr_reader :review_request, :user_finder def build_reviewers - reviewer_ids = review_request.users.map { |user| user_finder.user_id_for(user) }.compact + reviewer_ids = review_request.users.filter_map { |user| user_finder.user_id_for(user) } reviewer_ids.map do |reviewer_id| MergeRequestReviewer.new( diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb index c5d8da3be1c..0a92aee801d 100644 --- a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb @@ -18,6 +18,7 @@ module Gitlab review_requests = client.pull_request_review_requests(repo, merge_request.iid) review_requests[:merge_request_id] = merge_request.id + review_requests[:merge_request_iid] = merge_request.iid yield review_requests mark_merge_request_imported(merge_request) diff --git a/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb new file mode 100644 index 00000000000..347423b0e21 --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/reviews_importer.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewsImporter + include ParallelScheduling + + def initialize(...) + super + + @merge_requests_already_imported_cache_key = + "github-importer/merge_request/already-imported/#{project.id}" + end + + def importer_class + ReviewImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequestReview + end + + def sidekiq_worker_class + Gitlab::GithubImport::PullRequests::ImportReviewWorker + end + + def collection_method + :pull_request_reviews + end + + def object_type + :pull_request_review + end + + def id_for_already_imported_cache(review) + review[:id] + end + + # The worker can be interrupted, by rate limit for instance, + # in different situations. To avoid requesting already imported data, + # if the worker is interrupted: + # - before importing all reviews of a merge request + # The reviews page is cached with the `PageCounter`, by merge request. + # - before importing all merge requests reviews + # Merge requests that had all the reviews imported are cached with + # `mark_merge_request_reviews_imported` + def each_object_to_import(&_block) + each_review_page do |page, merge_request| + page.objects.each do |review| + review = review.to_h + + next if already_imported?(review) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + review[:merge_request_id] = merge_request.id + review[:merge_request_iid] = merge_request.iid + yield(review) + + mark_as_imported(review) + end + end + end + + private + + attr_reader :merge_requests_already_imported_cache_key + + def each_review_page + merge_requests_to_import.find_each do |merge_request| + # The page counter needs to be scoped by merge request to avoid skipping + # pages of reviews from already imported merge requests. + page_counter = PageCounter.new(project, page_counter_id(merge_request)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) + + client.each_page(collection_method, repo, merge_request.iid, options) do |page| + next unless page_counter.set(page.number) + + yield(page, merge_request) + end + + # Avoid unnecessary Redis cache keys after the work is done. + page_counter.expire! + mark_merge_request_reviews_imported(merge_request) + end + end + + # Returns only the merge requests that still have reviews to be imported. + def merge_requests_to_import + project.merge_requests.id_not_in(already_imported_merge_requests) + end + + def already_imported_merge_requests + Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key) + end + + def page_counter_id(merge_request) + "merge_request/#{merge_request.id}/#{collection_method}" + end + + def mark_merge_request_reviews_imported(merge_request) + Gitlab::Cache::Import::Caching.set_add( + merge_requests_already_imported_cache_key, + merge_request.id + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb b/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb deleted file mode 100644 index c56b391cbec..00000000000 --- a/lib/gitlab/github_import/importer/pull_requests_merged_by_importer.rb +++ /dev/null @@ -1,57 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class PullRequestsMergedByImporter - include ParallelScheduling - - def importer_class - PullRequestMergedByImporter - end - - def representation_class - Gitlab::GithubImport::Representation::PullRequest - end - - def sidekiq_worker_class - ImportPullRequestMergedByWorker - end - - def collection_method - :pull_requests_merged_by - end - - def object_type - :pull_request_merged_by - end - - def id_for_already_imported_cache(merge_request) - merge_request.id - end - - def each_object_to_import - merge_requests_to_import.find_each do |merge_request| - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - pull_request = client.pull_request(project.import_source, merge_request.iid) - yield(pull_request) - - mark_as_imported(merge_request) - end - end - - private - - # Returns only the merge requests that still have merged_by to be imported. - def merge_requests_to_import - project.merge_requests.id_not_in(already_imported_objects).with_state(:merged) - end - - def already_imported_objects - Gitlab::Cache::Import::Caching.values_from_set(already_imported_cache_key) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb b/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb deleted file mode 100644 index 543c29a21a0..00000000000 --- a/lib/gitlab/github_import/importer/pull_requests_reviews_importer.rb +++ /dev/null @@ -1,111 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GithubImport - module Importer - class PullRequestsReviewsImporter - include ParallelScheduling - - def initialize(...) - super - - @merge_requests_already_imported_cache_key = - "github-importer/merge_request/already-imported/#{project.id}" - end - - def importer_class - PullRequestReviewImporter - end - - def representation_class - Gitlab::GithubImport::Representation::PullRequestReview - end - - def sidekiq_worker_class - ImportPullRequestReviewWorker - end - - def collection_method - :pull_request_reviews - end - - def object_type - :pull_request_review - end - - def id_for_already_imported_cache(review) - review[:id] - end - - # The worker can be interrupted, by rate limit for instance, - # in different situations. To avoid requesting already imported data, - # if the worker is interrupted: - # - before importing all reviews of a merge request - # The reviews page is cached with the `PageCounter`, by merge request. - # - before importing all merge requests reviews - # Merge requests that had all the reviews imported are cached with - # `mark_merge_request_reviews_imported` - def each_object_to_import(&block) - each_review_page do |page, merge_request| - page.objects.each do |review| - review = review.to_h - - next if already_imported?(review) - - Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - - review[:merge_request_id] = merge_request.id - yield(review) - - mark_as_imported(review) - end - end - end - - private - - attr_reader :merge_requests_already_imported_cache_key - - def each_review_page - merge_requests_to_import.find_each do |merge_request| - # The page counter needs to be scoped by merge request to avoid skipping - # pages of reviews from already imported merge requests. - page_counter = PageCounter.new(project, page_counter_id(merge_request)) - repo = project.import_source - options = collection_options.merge(page: page_counter.current) - - client.each_page(collection_method, repo, merge_request.iid, options) do |page| - next unless page_counter.set(page.number) - - yield(page, merge_request) - end - - # Avoid unnecessary Redis cache keys after the work is done. - page_counter.expire! - mark_merge_request_reviews_imported(merge_request) - end - end - - # Returns only the merge requests that still have reviews to be imported. - def merge_requests_to_import - project.merge_requests.id_not_in(already_imported_merge_requests) - end - - def already_imported_merge_requests - Gitlab::Cache::Import::Caching.values_from_set(merge_requests_already_imported_cache_key) - end - - def page_counter_id(merge_request) - "merge_request/#{merge_request.id}/#{collection_method}" - end - - def mark_merge_request_reviews_imported(merge_request) - Gitlab::Cache::Import::Caching.set_add( - merge_requests_already_imported_cache_key, - merge_request.id - ) - end - end - end - end -end diff --git a/lib/gitlab/github_import/importer/releases_importer.rb b/lib/gitlab/github_import/importer/releases_importer.rb index 62d579fda08..2f210dafd0c 100644 --- a/lib/gitlab/github_import/importer/releases_importer.rb +++ b/lib/gitlab/github_import/importer/releases_importer.rb @@ -73,6 +73,13 @@ module Gitlab def model Release end + + def github_identifiers(release) + { + tag: release[:tag_name], + object_type: object_type + } + end end end end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index d7fe01e90f8..2654812b64a 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -66,13 +66,10 @@ module Gitlab true rescue ::Gitlab::Git::CommandError => e - if e.message !~ /repository not exported/ - project.create_wiki + return true if e.message.include?('repository not exported') - raise e - else - true - end + project.create_wiki + raise e end def wiki_url @@ -89,10 +86,8 @@ module Gitlab client_repository[:default_branch] end - def client_repository - strong_memoize(:client_repository) do - client.repository(project.import_source) - end + strong_memoize_attr def client_repository + client.repository(project.import_source) end end end diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb new file mode 100644 index 00000000000..52b211c92d6 --- /dev/null +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + # Used to calculate delay to spread sidekiq jobs on fetching records during import + # and upon job reschedule when the rate limit is reached + module JobDelayCalculator + # Default batch settings for parallel import (can be redefined in Importer/Worker classes) + def parallel_import_batch + { size: 1000, delay: 1.minute } + end + + private + + def calculate_job_delay(job_index) + multiplier = (job_index / parallel_import_batch[:size]) + + (multiplier * parallel_import_batch[:delay]) + 1.second + end + end + end +end diff --git a/lib/gitlab/github_import/markdown/attachment.rb b/lib/gitlab/github_import/markdown/attachment.rb index 1c814e34a39..e270cfba619 100644 --- a/lib/gitlab/github_import/markdown/attachment.rb +++ b/lib/gitlab/github_import/markdown/attachment.rb @@ -79,6 +79,22 @@ module Gitlab @url = url end + def part_of_project_blob?(import_source) + url.start_with?( + "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/blob" + ) + end + + def doc_belongs_to_project?(import_source) + url.start_with?( + "#{::Gitlab::GithubImport::MarkdownText.github_url}/#{import_source}/files" + ) + end + + def media? + url.start_with?(::Gitlab::GithubImport::MarkdownText::GITHUB_MEDIA_CDN) + end + def inspect "<#{self.class.name}: { name: #{name}, url: #{url} }>" end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index 4b54a77983d..cfc1ec526b0 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -3,6 +3,8 @@ module Gitlab module GithubImport module ParallelScheduling + include JobDelayCalculator + attr_reader :project, :client, :page_counter, :already_imported_cache_key, :job_waiter_cache_key, :job_waiter_remaining_cache_key @@ -85,14 +87,10 @@ module Gitlab def parallel_import raise 'Batch settings must be defined for parallel import' if parallel_import_batch.blank? - if Feature.enabled?(:improved_spread_parallel_import) - improved_spread_parallel_import - else - spread_parallel_import - end + spread_parallel_import end - def improved_spread_parallel_import + def spread_parallel_import enqueued_job_counter = 0 each_object_to_import do |object| @@ -108,33 +106,6 @@ module Gitlab job_waiter end - def spread_parallel_import - waiter = JobWaiter.new - - import_arguments = [] - - each_object_to_import do |object| - repr = object_representation(object) - - import_arguments << [project.id, repr.to_hash, waiter.key] - - waiter.jobs_remaining += 1 - end - - # rubocop:disable Scalability/BulkPerformWithContext - Gitlab::ApplicationContext.with_context(project: project) do - sidekiq_worker_class.bulk_perform_in( - 1.second, - import_arguments, - batch_size: parallel_import_batch[:size], - batch_delay: parallel_import_batch[:delay] - ) - end - # rubocop:enable Scalability/BulkPerformWithContext - - waiter - end - # The method that will be called for traversing through all the objects to # import, yielding them to the supplied block. def each_object_to_import @@ -228,11 +199,6 @@ module Gitlab raise NotImplementedError end - # Default batch settings for parallel import (can be redefined in Importer classes) - def parallel_import_batch - { size: 1000, delay: 1.minute } - end - def abort_on_failure false end @@ -274,12 +240,6 @@ module Gitlab JobWaiter.new(jobs_remaining, key) end end - - def calculate_job_delay(job_index) - multiplier = (job_index / parallel_import_batch[:size]) - - (multiplier * parallel_import_batch[:delay]) + 1.second - end end end end diff --git a/lib/gitlab/github_import/project_relation_type.rb b/lib/gitlab/github_import/project_relation_type.rb new file mode 100644 index 00000000000..a6e598172ee --- /dev/null +++ b/lib/gitlab/github_import/project_relation_type.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ProjectRelationType + CACHE_ORGS_EXPIRES_IN = 5.minutes + CACHE_USER_EXPIRES_IN = 1.hour + + def initialize(client) + @client = client + end + + def for(import_source) + namespace = import_source.split('/')[0] + if user?(namespace) + 'owned' + elsif organization?(namespace) + 'organization' + else + 'collaborated' + end + end + + private + + attr_reader :client + + def user?(namespace) + github_user_login == namespace + end + + def organization?(namespace) + github_org_logins.include? namespace + end + + def github_user_login + ::Rails.cache.fetch(cache_key('user_login'), expire_in: CACHE_USER_EXPIRES_IN) do + client.user(nil)[:login] + end + end + + def github_org_logins + ::Rails.cache.fetch(cache_key('organization_logins'), expires_in: CACHE_ORGS_EXPIRES_IN) do + logins = [] + client.each_object(:organizations) { |org| logins.push(org[:login]) } + logins + end + end + + def cache_key(subject) + ['github_import', Gitlab::CryptoHelper.sha256(client.octokit.access_token), subject].join('/') + end + end + end +end diff --git a/lib/gitlab/github_import/representation/collaborator.rb b/lib/gitlab/github_import/representation/collaborator.rb new file mode 100644 index 00000000000..fb58a572151 --- /dev/null +++ b/lib/gitlab/github_import/representation/collaborator.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class Collaborator + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :login, :role_name + + # Builds a user from a GitHub API response. + # + # collaborator - An instance of `Hash` containing the user & role details. + def self.from_api_response(collaborator, _additional_data = {}) + new( + id: collaborator[:id], + login: collaborator[:login], + role_name: collaborator[:role_name] + ) + end + + # Builds a user using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the user details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { + id: id, + login: login + } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/diff_note.rb b/lib/gitlab/github_import/representation/diff_note.rb index 9259d0295d5..0408b34bb02 100644 --- a/lib/gitlab/github_import/representation/diff_note.rb +++ b/lib/gitlab/github_import/representation/diff_note.rb @@ -120,7 +120,7 @@ module Gitlab def github_identifiers { note_id: note_id, - noteable_id: noteable_id, + noteable_iid: noteable_id, noteable_type: noteable_type } end diff --git a/lib/gitlab/github_import/representation/issue.rb b/lib/gitlab/github_import/representation/issue.rb index e878aeaf3b9..95a7c5ebf4b 100644 --- a/lib/gitlab/github_import/representation/issue.rb +++ b/lib/gitlab/github_import/representation/issue.rb @@ -79,7 +79,8 @@ module Gitlab def github_identifiers { iid: iid, - issuable_type: issuable_type + issuable_type: issuable_type, + title: title } end end diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 39a23c016ce..068d5cf9482 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -20,7 +20,11 @@ module Gitlab end def github_identifiers - { id: id } + { + id: id, + issuable_iid: issuable_id, + event: event + } end def issuable_type diff --git a/lib/gitlab/github_import/representation/lfs_object.rb b/lib/gitlab/github_import/representation/lfs_object.rb index cd614db2161..716e77bf401 100644 --- a/lib/gitlab/github_import/representation/lfs_object.rb +++ b/lib/gitlab/github_import/representation/lfs_object.rb @@ -33,7 +33,8 @@ module Gitlab def github_identifiers { - oid: oid + oid: oid, + size: size } end end diff --git a/lib/gitlab/github_import/representation/note.rb b/lib/gitlab/github_import/representation/note.rb index 14379e8a4e9..7a8bdfb1c64 100644 --- a/lib/gitlab/github_import/representation/note.rb +++ b/lib/gitlab/github_import/representation/note.rb @@ -76,7 +76,7 @@ module Gitlab def github_identifiers { note_id: note_id, - noteable_id: noteable_id, + noteable_iid: noteable_id, noteable_type: noteable_type } end diff --git a/lib/gitlab/github_import/representation/note_text.rb b/lib/gitlab/github_import/representation/note_text.rb index 505d7d805d3..70dd242303a 100644 --- a/lib/gitlab/github_import/representation/note_text.rb +++ b/lib/gitlab/github_import/representation/note_text.rb @@ -16,35 +16,35 @@ module Gitlab attr_reader :attributes - expose_attribute :record_db_id, :record_type, :text - - class << self - # Builds a note text representation from DB record of Note or Release. - # - # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model - def from_db_record(record) - check_record_class!(record) - - record_type = record.class.name - # only column for note is different along MODELS_ALLOWLIST - text = record.is_a?(::Note) ? record.note : record.description - new( - record_db_id: record.id, - record_type: record_type, - text: text - ) - end + expose_attribute :record_db_id, :record_type, :text, :iid, :tag, :noteable_type - def from_json_hash(raw_hash) - new Representation.symbolize_hash(raw_hash) - end + # Builds a note text representation from DB record of Note or Release. + # + # record - An instance of `Note`, `Release`, `Issue`, `MergeRequest` model + def self.from_db_record(record) + check_record_class!(record) - private + record_type = record.class.name + # only column for note is different along MODELS_ALLOWLIST + text = record.is_a?(::Note) ? record.note : record.description + new( + record_db_id: record.id, + record_type: record_type, + text: text, + iid: record.try(:iid), + tag: record.try(:tag), + noteable_type: record.try(:noteable_type) + ) + end - def check_record_class!(record) - raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class) - end + def self.from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + def self.check_record_class!(record) + raise ModelNotSupported, record.class.name if MODELS_ALLOWLIST.exclude?(record.class) end + private_class_method :check_record_class! # attributes - A Hash containing the event details. The keys of this # Hash (and any nested hashes) must be symbols. @@ -53,7 +53,22 @@ module Gitlab end def github_identifiers - { db_id: record_db_id } + { + db_id: record_db_id + }.merge(record_type_specific_attribute) + end + + private + + def record_type_specific_attribute + case record_type + when ::Release.name + { tag: tag } + when ::Issue.name, ::MergeRequest.name + { noteable_iid: iid } + when ::Note.name + { noteable_type: noteable_type } + end end end end diff --git a/lib/gitlab/github_import/representation/pull_request.rb b/lib/gitlab/github_import/representation/pull_request.rb index 4b8ae1f8eab..f26fa953773 100644 --- a/lib/gitlab/github_import/representation/pull_request.rb +++ b/lib/gitlab/github_import/representation/pull_request.rb @@ -111,7 +111,8 @@ module Gitlab def github_identifiers { iid: iid, - issuable_type: issuable_type + issuable_type: issuable_type, + title: title } end end diff --git a/lib/gitlab/github_import/representation/pull_request_review.rb b/lib/gitlab/github_import/representation/pull_request_review.rb index 8fb57ae89a4..0c6e281cd6d 100644 --- a/lib/gitlab/github_import/representation/pull_request_review.rb +++ b/lib/gitlab/github_import/representation/pull_request_review.rb @@ -9,7 +9,7 @@ module Gitlab attr_reader :attributes - expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :review_id + expose_attribute :author, :note, :review_type, :submitted_at, :merge_request_id, :merge_request_iid, :review_id # Builds a PullRequestReview from a GitHub API response. # @@ -19,6 +19,7 @@ module Gitlab new( merge_request_id: review[:merge_request_id], + merge_request_iid: review[:merge_request_iid], author: user, note: review[:body], review_type: review[:state], @@ -49,8 +50,8 @@ module Gitlab def github_identifiers { - review_id: review_id, - merge_request_id: merge_request_id + merge_request_iid: merge_request_iid, + review_id: review_id } end end diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb index 692004c4460..a6ec1d3178b 100644 --- a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb +++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes - expose_attribute :merge_request_id, :users + expose_attribute :merge_request_id, :merge_request_iid, :users class << self # Builds a list of requested reviewers from a GitHub API response. @@ -24,6 +24,7 @@ module Gitlab new( merge_request_id: review_requests[:merge_request_id], + merge_request_iid: review_requests[:merge_request_iid], users: users ) end @@ -37,7 +38,10 @@ module Gitlab end def github_identifiers - { merge_request_id: merge_request_id } + { + merge_request_iid: merge_request_iid, + requested_reviewers: users.pluck(:login) # rubocop: disable CodeReuse/ActiveRecord + } end end end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 77288b9fb98..0b883de8ed0 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -6,6 +6,7 @@ module Gitlab OPTIONAL_STAGES = { single_endpoint_issue_events_import: { label: 'Import issue and pull request events', + selected: false, details: <<-TEXT.split("\n").map(&:strip).join(' ') For example, opened or closed, renamed, and labeled or unlabeled. Time required to import these events depends on how many issues or pull requests your project has. @@ -13,17 +14,27 @@ module Gitlab }, single_endpoint_notes_import: { label: 'Use alternative comments import method', + selected: false, details: <<-TEXT.split("\n").map(&:strip).join(' ') The default method can skip some comments in large projects because of limitations of the GitHub API. TEXT }, attachments_import: { - label: 'Import Markdown attachments', + label: 'Import Markdown attachments (links)', + selected: false, details: <<-TEXT.split("\n").map(&:strip).join(' ') - Import Markdown attachments from repository comments, release posts, issue descriptions, + Import Markdown attachments (links) from repository comments, release posts, issue descriptions, and pull request descriptions. These can include images, text, or binary attachments. If not imported, links in Markdown to attachments break after you remove the attachments from GitHub. TEXT + }, + collaborators_import: { + label: 'Import collaborators', + selected: true, + details: <<-TEXT.split("\n").map(&:strip).join(' ') + Import direct repository collaborators who are not outside collaborators. + Imported collaborators who aren't members of the group you imported the project into consume seats on your GitLab instance. + TEXT } }.freeze @@ -32,6 +43,7 @@ module Gitlab { name: stage_name.to_s, label: s_(format("GitHubImport|%{text}", text: data[:label])), + selected: data[:selected], details: s_(format("GitHubImport|%{text}", text: data[:details])) } end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index b8751def08f..dd71edbd205 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -28,6 +28,9 @@ module Gitlab EMAIL_FOR_USERNAME_CACHE_KEY = 'github-import/user-finder/email-for-username/%s' + # The base cache key to use for caching inexistence of GitHub usernames. + INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY = 'github-import/user-finder/inexistence-of-username/%s' + # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` def initialize(project, client) @@ -113,12 +116,15 @@ module Gitlab cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username email = Gitlab::Cache::Import::Caching.read(cache_key) - unless email + if email.blank? && !github_username_inexists?(username) user = client.user(username) email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user end email + rescue ::Octokit::NotFound + cache_github_username_inexistence(username) + nil end def cached_id_for_github_id(id) @@ -190,6 +196,18 @@ module Gitlab Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT end end + + def github_username_inexists?(username) + cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username + + Gitlab::Cache::Import::Caching.read(cache_key) == 'true' + end + + def cache_github_username_inexistence(username) + cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username + + Gitlab::Cache::Import::Caching.write(cache_key, true) + end end end end diff --git a/lib/gitlab/gitlab_import/client.rb b/lib/gitlab/gitlab_import/client.rb deleted file mode 100644 index 86474159f8b..00000000000 --- a/lib/gitlab/gitlab_import/client.rb +++ /dev/null @@ -1,89 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitlabImport - class Client - attr_reader :client, :api - - PER_PAGE = 100 - - def initialize(access_token) - @client = ::OAuth2::Client.new( - config.app_id, - config.app_secret, - gitlab_options - ) - - if access_token - @api = OAuth2::AccessToken.from_hash(@client, access_token: access_token) - end - end - - def authorize_url(redirect_uri) - client.auth_code.authorize_url({ - redirect_uri: redirect_uri, - scope: "api" - }) - end - - def get_token(code, redirect_uri) - client.auth_code.get_token(code, redirect_uri: redirect_uri).token - end - - def user - api.get("/api/v4/user").parsed - end - - def issues(project_identifier, **kwargs) - lazy_page_iterator(**kwargs) do |page, per_page| - api.get("/api/v4/projects/#{project_identifier}/issues?per_page=#{per_page}&page=#{page}").parsed - end - end - - def issue_comments(project_identifier, issue_id, **kwargs) - lazy_page_iterator(**kwargs) do |page, per_page| - api.get("/api/v4/projects/#{project_identifier}/issues/#{issue_id}/notes?per_page=#{per_page}&page=#{page}").parsed - end - end - - def project(id) - api.get("/api/v4/projects/#{id}").parsed - end - - def projects(**kwargs) - lazy_page_iterator(**kwargs) do |page, per_page| - api.get("/api/v4/projects?per_page=#{per_page}&page=#{page}&simple=true&membership=true").parsed - end - end - - private - - def lazy_page_iterator(starting_page: 1, page_limit: nil, per_page: PER_PAGE) - Enumerator.new do |y| - page = starting_page - page_limit = (starting_page - 1) + page_limit if page_limit - - loop do - items = yield(page, per_page) - - items.each do |item| - y << item - end - - break if items.empty? || items.size < per_page || (page_limit && page >= page_limit) - - page += 1 - end - end - end - - def config - Gitlab::Auth::OAuth::Provider.config_for('gitlab') - end - - def gitlab_options - OmniAuth::Strategies::GitLab.default_options[:client_options].to_h.symbolize_keys - end - end - end -end diff --git a/lib/gitlab/gitlab_import/importer.rb b/lib/gitlab/gitlab_import/importer.rb deleted file mode 100644 index 0cd33b9f5b7..00000000000 --- a/lib/gitlab/gitlab_import/importer.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitlabImport - class Importer - attr_reader :project, :client - - def initialize(project) - @project = project - import_data = project.import_data - if import_data && import_data.credentials && import_data.credentials[:password] - @client = Client.new(import_data.credentials[:password]) - @formatter = Gitlab::ImportFormatter.new - else - raise Projects::ImportService::Error, "Unable to find project import data credentials for project ID: #{@project.id}" - end - end - - def execute - ActiveRecord::Base.no_touching do - project_identifier = CGI.escape(project.import_source) - - # Issues && Comments - issues = client.issues(project_identifier) - - issues.each do |issue| - body = [@formatter.author_line(issue["author"]["name"])] - body << issue["description"] - - comments = client.issue_comments(project_identifier, issue["iid"]) - - if comments.any? - body << @formatter.comments_header - end - - comments.each do |comment| - body << @formatter.comment(comment["author"]["name"], comment["created_at"], comment["body"]) - end - - project.issues.create!( - iid: issue["iid"], - description: body.join, - title: issue["title"], - state: issue["state"], - updated_at: issue["updated_at"], - author_id: gitlab_user_id(project, issue["author"]["id"]), - confidential: issue["confidential"] - ) - end - end - - true - end - - private - - # rubocop: disable CodeReuse/ActiveRecord - def gitlab_user_id(project, gitlab_id) - user_id = User.by_provider_and_extern_uid(:gitlab, gitlab_id).select(:id).first&.id - user_id || project.creator_id - end - # rubocop: enable CodeReuse/ActiveRecord - end - end -end diff --git a/lib/gitlab/gitlab_import/project_creator.rb b/lib/gitlab/gitlab_import/project_creator.rb deleted file mode 100644 index 35feea17351..00000000000 --- a/lib/gitlab/gitlab_import/project_creator.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module GitlabImport - class ProjectCreator - attr_reader :repo, :namespace, :current_user, :session_data - - def initialize(repo, namespace, current_user, session_data) - @repo = repo - @namespace = namespace - @current_user = current_user - @session_data = session_data - end - - def execute - ::Projects::CreateService.new( - current_user, - name: repo["name"], - path: repo["path"], - description: repo["description"], - namespace_id: namespace.id, - visibility_level: Gitlab::VisibilityLevel.level_value(repo["visibility"]), - import_type: "gitlab", - import_source: repo["path_with_namespace"], - import_url: repo["http_url_to_repo"].sub("://", "://oauth2:#{@session_data[:gitlab_access_token]}@") - ).execute - end - end - end -end diff --git a/lib/gitlab/gl_repository.rb b/lib/gitlab/gl_repository.rb index d123989ef8e..268d5d3e564 100644 --- a/lib/gitlab/gl_repository.rb +++ b/lib/gitlab/gl_repository.rb @@ -34,8 +34,9 @@ module Gitlab DESIGN = ::Gitlab::GlRepository::RepoType.new( name: :design, access_checker_class: ::Gitlab::GitAccessDesign, - repository_resolver: -> (project) { ::DesignManagement::Repository.new(project) }, - suffix: :design + repository_resolver: -> (project) { project.design_management_repository.repository }, + suffix: :design, + container_class: DesignManagement::Repository ).freeze TYPES = { diff --git a/lib/gitlab/gl_repository/repo_type.rb b/lib/gitlab/gl_repository/repo_type.rb index 7792ef55b28..26b0ff86f67 100644 --- a/lib/gitlab/gl_repository/repo_type.rb +++ b/lib/gitlab/gl_repository/repo_type.rb @@ -55,11 +55,11 @@ module Gitlab def repository_for(container) return unless container - repository_resolver.call(container) + repository_resolver.call(select_container(container)) end def project_for(container) - return container unless project_resolver + return select_container(container) unless project_resolver project_resolver.call(container) end @@ -74,6 +74,10 @@ module Gitlab private + def select_container(container) + container.is_a?(::DesignManagement::Repository) ? container.project : container + end + def default_container_class Project end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index c9766ee095a..904a2ccc79b 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -49,6 +49,7 @@ module Gitlab gon.ee = Gitlab.ee? gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? + gon.uf_error_prefix = ::Gitlab::Utils::ErrorMessage::UF_ERROR_PREFIX if current_user gon.current_user_id = current_user.id @@ -56,18 +57,19 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative - gon.use_new_navigation = Feature.enabled?(:super_sidebar_nav, current_user) && current_user&.use_new_navigation + gon.use_new_navigation = NavHelper.show_super_sidebar?(current_user) end # Initialize gon.features with any flags that should be # made globally available to the frontend push_frontend_feature_flag(:usage_data_api, type: :ops) push_frontend_feature_flag(:security_auto_fix) - push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:integration_slack_app_notifications) - push_frontend_feature_flag(:full_path_project_search, current_user) + push_frontend_feature_flag(:super_sidebar_peek, current_user) + push_frontend_feature_flag(:unbatch_graphql_queries, current_user) + # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 + push_frontend_feature_flag(:remove_monitor_metrics) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb index 0465f01f7f5..767c282d62e 100644 --- a/lib/gitlab/grape_logging/loggers/response_logger.rb +++ b/lib/gitlab/grape_logging/loggers/response_logger.rb @@ -8,7 +8,14 @@ module Gitlab return {} unless Feature.enabled?(:log_response_length) response_bytes = 0 - response.each { |resp| response_bytes += resp.to_s.bytesize } + + case response + when String + response_bytes = response.bytesize + else + response.each { |resp| response_bytes += resp.to_s.bytesize } + end + { response_bytes: response_bytes } diff --git a/lib/gitlab/graphql/authorize/authorize_resource.rb b/lib/gitlab/graphql/authorize/authorize_resource.rb index 983bdb9c0a2..e3548b97ebf 100644 --- a/lib/gitlab/graphql/authorize/authorize_resource.rb +++ b/lib/gitlab/graphql/authorize/authorize_resource.rb @@ -45,8 +45,8 @@ module Gitlab end end - def find_object(*args) - raise NotImplementedError, "Implement #find_object in #{self.class.name}" + def find_object(id:) + GitlabSchema.find_by_gid(id) end def authorized_find!(*args, **kwargs) diff --git a/lib/gitlab/graphql/deprecations/deprecation.rb b/lib/gitlab/graphql/deprecations/deprecation.rb index 7f4cea7c635..dfcca5ee75b 100644 --- a/lib/gitlab/graphql/deprecations/deprecation.rb +++ b/lib/gitlab/graphql/deprecations/deprecation.rb @@ -9,7 +9,7 @@ module Gitlab REASONS = { REASON_RENAMED => 'This was renamed.', - REASON_ALPHA => 'This feature is in Alpha. It can be changed or removed at any time.' + REASON_ALPHA => 'This feature is an Experiment. It can be changed or removed at any time.' }.freeze include ActiveModel::Validations @@ -27,7 +27,7 @@ module Gitlab return unless options if alpha - raise ArgumentError, '`alpha` and `deprecated` arguments cannot be passed at the same time' \ + raise ArgumentError, '`experiment` and `deprecated` arguments cannot be passed at the same time' \ if deprecated options[:reason] = :alpha diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader.rb new file mode 100644 index 00000000000..69056e87091 --- /dev/null +++ b/lib/gitlab/graphql/loaders/lazy_relation_loader.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class LazyRelationLoader + class << self + attr_accessor :model, :association + + # Automatically register the inheriting + # classes to GitlabSchema as lazy objects. + def inherited(klass) + GitlabSchema.lazy_resolve(klass, :load) + end + end + + def initialize(query_ctx, object, **kwargs) + @query_ctx = query_ctx + @object = object + @kwargs = kwargs + + query_ctx[loader_cache_key] ||= Registry.new(relation(**kwargs)) + query_ctx[loader_cache_key].register(object) + end + + # Returns an instance of `RelationProxy` for the object (parent model). + # The returned object behaves like an Active Record relation to support + # keyset pagination. + def load + case reflection.macro + when :has_many + relation_proxy + when :has_one + relation_proxy.last + else + raise 'Not supported association type!' + end + end + + private + + attr_reader :query_ctx, :object, :kwargs + + delegate :model, :association, to: :"self.class" + + # Implement this one if you want to filter the relation + def relation(**) + base_relation + end + + def loader_cache_key + @loader_cache_key ||= self.class.name.to_s + kwargs.sort.to_s + end + + def base_relation + placeholder_record.association(association).scope + end + + # This will only work for HasMany and HasOne associations for now + def placeholder_record + model.new(reflection.active_record_primary_key => 0) + end + + def reflection + model.reflections[association.to_s] + end + + def relation_proxy + RelationProxy.new(object, query_ctx[loader_cache_key]) + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb new file mode 100644 index 00000000000..ab2b2bd4dc2 --- /dev/null +++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/registry.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class LazyRelationLoader + class Registry + PrematureQueryExecutionTriggered = Class.new(RuntimeError) + # Following methods are Active Record kicker methods which fire SQL query. + # We can support some of them with TopNLoader but for now restricting their use + # as we don't have a use case. + PROHIBITED_METHODS = ( + ActiveRecord::FinderMethods.instance_methods(false) + + ActiveRecord::Calculations.instance_methods(false) + ).to_set.freeze + + def initialize(relation) + @parents = [] + @relation = relation + @records = [] + @loaded = false + end + + def register(object) + @parents << object + end + + def method_missing(method_name, ...) + raise PrematureQueryExecutionTriggered if PROHIBITED_METHODS.include?(method_name) + + result = relation.public_send(method_name, ...) # rubocop:disable GitlabSecurity/PublicSend + + if result.is_a?(ActiveRecord::Relation) # Spawn methods generate a new relation (e.g. where, limit) + @relation = result + + return self + end + + result + end + + def respond_to_missing?(method_name, include_private = false) + relation.respond_to?(method_name, include_private) + end + + def load + return records if loaded + + @loaded = true + @records = TopNLoader.load(relation, parents) + end + + def for(object) + load.select { |record| record[foreign_key] == object[active_record_primary_key] } + .tap { |records| set_inverse_of(object, records) } + end + + private + + attr_reader :parents, :relation, :records, :loaded + + delegate :proxy_association, to: :relation, private: true + delegate :reflection, to: :proxy_association, private: true + delegate :active_record_primary_key, :foreign_key, to: :reflection, private: true + + def set_inverse_of(object, records) + records.each do |record| + object.association(reflection.name).set_inverse_instance(record) + end + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb new file mode 100644 index 00000000000..bab2a272fb0 --- /dev/null +++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/relation_proxy.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Loaders + class LazyRelationLoader + # Proxies all the method calls to Registry instance. + # The main purpose of having this is that calling load + # on an instance of this class will only return the records + # associated with the main Active Record model. + class RelationProxy + def initialize(object, registry) + @object = object + @registry = registry + end + + def load + registry.for(object) + end + alias_method :to_a, :load + + def last(limit = 1) + result = registry.limit(limit) + .reverse_order! + .for(object) + + return result.first if limit == 1 # This is the Active Record behavior + + result + end + + private + + attr_reader :registry, :object + + # Delegate everything to registry + def method_missing(method_name, ...) + result = registry.public_send(method_name, ...) # rubocop:disable GitlabSecurity/PublicSend + + return self if result == registry + + result + end + + def respond_to_missing?(method_name, include_private = false) + registry.respond_to?(method_name, include_private) + end + end + end + end + end +end diff --git a/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb b/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb new file mode 100644 index 00000000000..6404148832b --- /dev/null +++ b/lib/gitlab/graphql/loaders/lazy_relation_loader/top_n_loader.rb @@ -0,0 +1,83 @@ +# frozen_string_literal: true + +# rubocop:disable CodeReuse/ActiveRecord +module Gitlab + module Graphql + module Loaders + class LazyRelationLoader + # Loads the top-n records for each given parent record. + # For example; if you want to load only 5 confidential issues ordered by + # their updated_at column per project for a list of projects by issuing only a single + # SQL query then this class can help you. + # Note that the limit applies per parent record which means that if you apply limit as 5 + # for 10 projects, this loader will load 50 records in total. + class TopNLoader + def self.load(original_relation, parents) + new(original_relation, parents).load + end + + def initialize(original_relation, parents) + @original_relation = original_relation + @parents = parents + end + + def load + klass.select(klass.arel_table[Arel.star]) + .from(from) + .joins("JOIN LATERAL (#{lateral_relation.to_sql}) AS #{klass.arel_table.name} ON true") + .includes(original_includes) + .preload(original_preload) + .eager_load(original_eager_load) + .load + end + + private + + attr_reader :original_relation, :parents + + delegate :proxy_association, to: :original_relation, private: true + delegate :reflection, to: :proxy_association, private: true + delegate :klass, :foreign_key, :active_record, :active_record_primary_key, + to: :reflection, private: true + + # This only works for HasMany and HasOne. + def lateral_relation + original_relation + .unscope(where: foreign_key) # unscoping the where condition generated for the placeholder_record. + .where(klass.arel_table[foreign_key].eq(active_record.arel_table[active_record_primary_key])) + end + + def from + grouping_arel_node.as("#{active_record.arel_table.name}(#{active_record.primary_key})") + end + + def grouping_arel_node + Arel::Nodes::Grouping.new(id_list_arel_node) + end + + def id_list_arel_node + parent_ids.map { |id| [id] } + .then { |ids| Arel::Nodes::ValuesList.new(ids) } + end + + def parent_ids + parents.pluck(active_record.primary_key) + end + + def original_includes + original_relation.includes_values + end + + def original_preload + original_relation.preload_values + end + + def original_eager_load + original_relation.eager_load_values + end + end + end + end + end +end +# rubocop:enable CodeReuse/ActiveRecord diff --git a/lib/gitlab/graphql/pagination/connections.rb b/lib/gitlab/graphql/pagination/connections.rb index 965c01dd02f..df1231b005f 100644 --- a/lib/gitlab/graphql/pagination/connections.rb +++ b/lib/gitlab/graphql/pagination/connections.rb @@ -14,6 +14,10 @@ module Gitlab Gitlab::Graphql::Pagination::Keyset::Connection) schema.connections.add( + Gitlab::Graphql::Loaders::LazyRelationLoader::RelationProxy, + Gitlab::Graphql::Pagination::Keyset::Connection) + + schema.connections.add( Gitlab::Graphql::ExternallyPaginatedArray, Gitlab::Graphql::Pagination::ExternallyPaginatedArrayConnection) diff --git a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb index 45f90de2f17..1c21d286187 100644 --- a/lib/gitlab/graphql/project/dast_profile_connection_extension.rb +++ b/lib/gitlab/graphql/project/dast_profile_connection_extension.rb @@ -12,9 +12,12 @@ module Gitlab def preload_authorizations(dast_profiles) return unless dast_profiles - projects = dast_profiles.map(&:project) - users = dast_profiles.filter_map { |dast_profile| dast_profile.dast_profile_schedule&.owner } - Preloaders::UsersMaxAccessLevelInProjectsPreloader.new(projects: projects, users: users).execute + project_users = dast_profiles.group_by(&:project).transform_values do |project_profiles| + project_profiles + .filter_map { |profile| profile.dast_profile_schedule&.owner } + .uniq + end + Preloaders::UsersMaxAccessLevelByProjectPreloader.new(project_users: project_users).execute end end end diff --git a/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb new file mode 100644 index 00000000000..851750163af --- /dev/null +++ b/lib/gitlab/graphql/subscriptions/action_cable_with_load_balancing.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Subscriptions + class ActionCableWithLoadBalancing < ::GraphQL::Subscriptions::ActionCableSubscriptions + extend ::Gitlab::Utils::Override + include Gitlab::Database::LoadBalancing::WalTrackingSender + include Gitlab::Database::LoadBalancing::WalTrackingReceiver + + KEY_PAYLOAD = 'gql_payload' + KEY_WAL_LOCATIONS = 'wal_locations' + + override :execute_all + def execute_all(event, object) + super(event, { + KEY_WAL_LOCATIONS => current_wal_locations, + KEY_PAYLOAD => object + }) + end + + # We fall back to the primary in case no replica is sufficiently caught up. + override :execute_update + def execute_update(subscription_id, event, object) + # Make sure we do not accidentally try to unwrap messages that are not wrapped. + # This could in theory happen if workers roll over where some send wrapped payload + # and others expect the original payload. + return super(subscription_id, event, object) unless wrapped_payload?(object) + + wal_locations = object[KEY_WAL_LOCATIONS] + ::Gitlab::Database::LoadBalancing::Session.current.use_primary! if use_primary?(wal_locations) + + super(subscription_id, event, object[KEY_PAYLOAD]) + end + + private + + def wrapped_payload?(object) + object.try(:key?, KEY_PAYLOAD) + end + + def use_primary?(wal_locations) + wal_locations.blank? || !databases_in_sync?(wal_locations) + end + + # We stringify keys since otherwise the graphql-ruby serializer will inject additional metadata + # to keep track of which keys used to be symbols. + def current_wal_locations + wal_locations_by_db_name&.stringify_keys + end + end + end + end +end diff --git a/lib/gitlab/harbor/client.rb b/lib/gitlab/harbor/client.rb index ee40725ba95..380e4e42bc7 100644 --- a/lib/gitlab/harbor/client.rb +++ b/lib/gitlab/harbor/client.rb @@ -14,9 +14,9 @@ module Gitlab @integration = integration end - def ping - options = { headers: headers.merge!('Accept': 'text/plain') } - response = Gitlab::HTTP.get(url('ping'), options) + def check_project_availability + options = { headers: headers.merge!('Accept': 'application/json') } + response = Gitlab::HTTP.head(url("projects?project_name=#{integration.project_name}"), options) { success: response.success? } end diff --git a/lib/gitlab/hook_data/base_builder.rb b/lib/gitlab/hook_data/base_builder.rb index e5bae61ae4e..4a81f6b8a0e 100644 --- a/lib/gitlab/hook_data/base_builder.rb +++ b/lib/gitlab/hook_data/base_builder.rb @@ -5,15 +5,14 @@ module Gitlab class BaseBuilder attr_accessor :object - MARKDOWN_SIMPLE_IMAGE = %r{ - #{::Gitlab::Regex.markdown_code_or_html_blocks} - | - (?<image> - ! - \[(?<title>[^\n]*?)\] - \((?<url>(?!(https?://|//))[^\n]+?)\) - ) - }mx.freeze + MARKDOWN_SIMPLE_IMAGE = + "#{::Gitlab::Regex.markdown_code_or_html_blocks_untrusted}" \ + '|' \ + '(?P<image>' \ + '!' \ + '\[(?P<title>[^\n]*?)\]' \ + '\((?P<url>(?P<https>(https?://|//)?)[^\n]+?)\)' \ + ')'.freeze def initialize(object) @object = object @@ -37,15 +36,18 @@ module Gitlab def absolute_image_urls(markdown_text) return markdown_text unless markdown_text.present? - markdown_text.gsub(MARKDOWN_SIMPLE_IMAGE) do - if $~[:image] - url = $~[:url] + regex = Gitlab::UntrustedRegexp.new(MARKDOWN_SIMPLE_IMAGE, multiline: false) + return markdown_text unless regex.match?(markdown_text) + + regex.replace_gsub(markdown_text) do |match| + if match[:image] && !match[:https] + url = match[:url] url = "#{uploads_prefix}#{url}" if url.start_with?('/uploads') url = "/#{url}" unless url.start_with?('/') - "![#{$~[:title]}](#{Gitlab.config.gitlab.url}#{url})" + "![#{match[:title]}](#{Gitlab.config.gitlab.url}#{url})" else - $~[0] + match.to_s end end end diff --git a/lib/gitlab/http_connection_adapter.rb b/lib/gitlab/http_connection_adapter.rb index c6f9f2df299..afb740a902b 100644 --- a/lib/gitlab/http_connection_adapter.rb +++ b/lib/gitlab/http_connection_adapter.rb @@ -24,11 +24,18 @@ module Gitlab override :connection def connection - @uri, hostname = validate_url!(uri) + result = validate_url_with_proxy!(uri) + @uri = result.uri + hostname = result.hostname http = super http.hostname_override = hostname if hostname + unless result.use_proxy + http.proxy_from_env = false + http.proxy_address = nil + end + gitlab_http = Gitlab::NetHttpAdapter.new(http.address, http.port) http.instance_variables.each do |variable| @@ -40,12 +47,13 @@ module Gitlab private - def validate_url!(url) - Gitlab::UrlBlocker.validate!(url, allow_local_network: allow_local_requests?, - allow_localhost: allow_local_requests?, - allow_object_storage: allow_object_storage?, - dns_rebind_protection: dns_rebind_protection?, - schemes: %w[http https]) + def validate_url_with_proxy!(url) + Gitlab::UrlBlocker.validate_url_with_proxy!( + url, allow_local_network: allow_local_requests?, + allow_localhost: allow_local_requests?, + allow_object_storage: allow_object_storage?, + dns_rebind_protection: dns_rebind_protection?, + schemes: %w[http https]) rescue Gitlab::UrlBlocker::BlockedUrlError => e raise Gitlab::HTTP::BlockedUrlError, "URL is blocked: #{e.message}" end @@ -59,8 +67,6 @@ module Gitlab end def dns_rebind_protection? - return false if Gitlab.http_proxy_env? - Gitlab::CurrentSettings.dns_rebinding_protection_enabled? end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 8fe5868ca57..fa85d839927 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 34, - 'de' => 16, + 'da_DK' => 31, + 'de' => 15, 'en' => 100, 'eo' => 0, - 'es' => 33, + 'es' => 31, 'fil_PH' => 0, - 'fr' => 99, + 'fr' => 98, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 31, - 'ko' => 20, + 'ja' => 77, + 'ko' => 18, 'nb_NO' => 23, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 91, - 'ru' => 26, - 'si_LK' => 11, - 'tr_TR' => 10, - 'uk' => 55, - 'zh_CN' => 98, + 'pt_BR' => 56, + 'ro_RO' => 84, + 'ru' => 24, + 'si_LK' => 10, + 'tr_TR' => 9, + 'uk' => 54, + 'zh_CN' => 99, 'zh_HK' => 1, - 'zh_TW' => 98 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS @@ -118,12 +118,17 @@ module Gitlab end def setup(domain:, default_locale:) + custom_pluralization setup_repositories(domain) setup_default_locale(default_locale) end private + def custom_pluralization + Gitlab::I18n::Pluralization.install_on(FastGettext) + end + def setup_repositories(domain) translation_repositories = [ (po_repository(domain, 'jh/locale') if Gitlab.jh?), diff --git a/lib/gitlab/i18n/pluralization.rb b/lib/gitlab/i18n/pluralization.rb new file mode 100644 index 00000000000..5d4a05f1fd1 --- /dev/null +++ b/lib/gitlab/i18n/pluralization.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Gitlab + module I18n + # Pluralization formulas per locale used by FastGettext via: + # `FastGettext.pluralisation_rule.call(count)`. + module Pluralization + # rubocop:disable all + MAP = { + "bg" => ->(n) { (n != 1) }, + "cs_CZ" => ->(n) { (n==1) ? 0 : (n>=2 && n<=4) ? 1 : 3 }, + "da_DK" => ->(n) { (n != 1) }, + "de" => ->(n) { (n != 1) }, + "en" => ->(n) { (n != 1) }, + "eo" => ->(n) { (n != 1) }, + "es" => ->(n) { (n != 1) }, + "fil_PH" => ->(n) { (n > 1) }, + "fr" => ->(n) { (n > 1) }, + "gl_ES" => ->(n) { (n != 1) }, + "id_ID" => ->(n) { 0 }, + "it" => ->(n) { (n != 1) }, + "ja" => ->(n) { 0 }, + "ko" => ->(n) { 0 }, + "nb_NO" => ->(n) { (n != 1) }, + "nl_NL" => ->(n) { (n != 1) }, + "pl_PL" => ->(n) { (n==1 ? 0 : (n%10>=2 && n%10<=4) && (n%100<12 || n%100>14) ? 1 : n!=1 && (n%10>=0 && n%10<=1) || (n%10>=5 && n%10<=9) || (n%100>=12 && n%100<=14) ? 2 : 3) }, + "pt_BR" => ->(n) { (n != 1) }, + "ro_RO" => ->(n) { (n==1 ? 0 : (n==0 || (n%100>0 && n%100<20)) ? 1 : 2) }, + "ru" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) }, + "si_LK" => ->(n) { (n != 1) }, + "tr_TR" => ->(n) { (n != 1) }, + "uk" => ->(n) { ((n%10==1 && n%100!=11) ? 0 : ((n%10 >= 2 && n%10 <=4 && (n%100 < 12 || n%100 > 14)) ? 1 : ((n%10 == 0 || (n%10 >= 5 && n%10 <=9)) || (n%100 >= 11 && n%100 <= 14)) ? 2 : 3)) }, + "zh_CN" => ->(n) { 0 }, + "zh_HK" => ->(n) { 0 }, + "zh_TW" => ->(n) { 0 } + }.freeze + # rubocop:enable + + NOT_FOUND_ERROR = lambda do |locale| + po = File.expand_path("../../../locale/#{locale}/gitlab.po", __dir__) + + forms = File.read(po)[/Plural-Forms:.*; plural=(.*?);\\n/, 1] if File.exist?(po) + suggestion = <<~TEXT if forms + Add the following line to #{__FILE__}: + + MAP = { + ... + "#{locale}" => ->(n) { #{forms} }, + ... + }.freeze + + This rule was extracted from #{po}. + TEXT + + raise ArgumentError, <<~MESSAGE + Missing pluralization rule for locale #{locale.inspect}. + + #{suggestion} + MESSAGE + end + + def self.call(count) + locale = FastGettext.locale + + MAP.fetch(locale, &NOT_FOUND_ERROR).call(count) + end + + def self.install_on(klass) + klass.extend(FastGettextClassMethods) + end + + module FastGettextClassMethods + # FastGettext allows to set the rule via + # `FastGettext.pluralisation_rule=` which is on thread-level. + # + # Because we are patching FastGettext at boot time per thread values + # won't work so we have to override the method implementation. + # + # `FastGettext.pluralisation_rule=` has now no effect. + def pluralisation_rule + Gitlab::I18n::Pluralization + end + end + end + end +end diff --git a/lib/gitlab/import/errors.rb b/lib/gitlab/import/errors.rb new file mode 100644 index 00000000000..b9e8c9135fd --- /dev/null +++ b/lib/gitlab/import/errors.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Import + module Errors + # Merges all nested subrelation errors into base errors object. + # + # @example + # issue = Project.last.issues.new( + # title: 'test', + # author: User.first, + # notes: [Note.new( + # award_emoji: [AwardEmoji.new(name: 'test')] + # )]) + # + # issue.validate + # issue.errors.full_messages + # => ["Notes is invalid"] + # + # Gitlab::Import::Errors.merge_nested_errors(issue) + # issue.errors.full_messages + # => ["Notes is invalid", + # "Award emoji is invalid", + # "Awardable can't be blank", + # "Name is not a valid emoji name", + # ... + # ] + def self.merge_nested_errors(object) + object.errors.each do |error| + association = object.class.reflect_on_association(error.attribute) + + next unless association&.collection? + + records = object.public_send(error.attribute).select(&:invalid?) # rubocop: disable GitlabSecurity/PublicSend + + records.each do |record| + merge_nested_errors(record) + + object.errors.merge!(record.errors) + end + end + end + end + end +end diff --git a/lib/gitlab/import/import_failure_service.rb b/lib/gitlab/import/import_failure_service.rb index bebd64b29a9..714d9b3edd9 100644 --- a/lib/gitlab/import/import_failure_service.rb +++ b/lib/gitlab/import/import_failure_service.rb @@ -50,10 +50,10 @@ module Gitlab def execute track_exception - persist_failure - - track_metrics if metrics - import_state.mark_as_failed(exception.message) if fail_import + persist_failure.tap do + import_state.mark_as_failed(exception.message) if fail_import + track_metrics if metrics + end end private diff --git a/lib/gitlab/import/metrics.rb b/lib/gitlab/import/metrics.rb index 7a0cf1682a6..8263df3dc37 100644 --- a/lib/gitlab/import/metrics.rb +++ b/lib/gitlab/import/metrics.rb @@ -32,6 +32,14 @@ module Gitlab return unless project.github_import? track_usage_event(:github_import_project_failure, project.id) + track_import_state('github', 'Import::GithubService') + end + + def track_canceled_import + return unless project.github_import? + + track_usage_event(:github_import_project_cancelled, project.id) + track_import_state('github', 'Import::GithubService') end def issues_counter @@ -75,7 +83,25 @@ module Gitlab def track_finish_metric return unless project.github_import? - track_usage_event(:github_import_project_success, project.id) + track_import_state('github', 'Import::GithubService') + + case project.beautified_import_status_name + when 'partially completed' + track_usage_event(:github_import_project_partially_completed, project.id) + when 'completed' + track_usage_event(:github_import_project_success, project.id) + end + end + + def track_import_state(type, category) + Gitlab::Tracking.event( + category, + 'create', + label: "#{type}_import_project_state", + project: project, + import_type: type, + state: project.beautified_import_status_name + ) end end end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 8843b4f5755..dea989931c7 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -3,7 +3,8 @@ module Gitlab module ImportExport class AttributesFinder - attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders + attr_reader :tree, :included_attributes, :excluded_attributes, :methods, :preloads, :export_reorders, + :import_only_tree def initialize(config:) @tree = config[:tree] || {} @@ -13,13 +14,16 @@ module Gitlab @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} @include_if_exportable = config[:include_if_exportable] || {} + @import_only_tree = config[:import_only_tree] || {} end def find_root(model_key) find(model_key, @tree[model_key]) end - def find_relations_tree(model_key) + def find_relations_tree(model_key, include_import_only_tree: false) + return @tree[model_key].deep_merge(@import_only_tree[model_key] || {}) if include_import_only_tree + @tree[model_key] end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index 8c7a6c13246..889cab88de4 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -80,7 +80,7 @@ module Gitlab # Deep traverse relations tree to build a list of allowed model relations def build_associations - stack = @attributes_finder.tree.to_a + stack = @attributes_finder.tree.deep_merge(@attributes_finder.import_only_tree).to_a while stack.any? model_name, relations = stack.pop diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index e3813070aa4..3d96e891797 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -295,6 +295,13 @@ module Gitlab end def unique_relation? + # this guard is necessary because + # when multiple approval_project_rules_protected_branch referenced the same protected branch + # or approval_project_rules_user referenced the same user + # the different instances were squashed into one + # because this method returned true for reason that needs investigation + return if @relation_sym == :approval_rules + strong_memoize(:unique_relation) do importable_foreign_key.present? && (has_unique_index_on_importable_fk? || uses_importable_fk_as_primary_key?) diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 77b85fc9f15..986191bdb6b 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -17,6 +17,8 @@ module Gitlab BATCH_SIZE = 100 MIN_RECORDS_SIZE = 1 + attr_reader :invalid_subrelations + # @param relation_object [Object] Object of a project/group, e.g. an issue # @param relation_key [String] Name of the object association to group/project, e.g. :issues # @param relation_definition [Hash] Object subrelations as defined in import_export.yml @@ -43,14 +45,11 @@ module Gitlab relation_object.save! save_subrelations - ensure - log_invalid_subrelations end private - attr_reader :relation_object, :relation_key, :relation_definition, - :importable, :collection_subrelations, :invalid_subrelations + attr_reader :relation_object, :relation_key, :relation_definition, :importable, :collection_subrelations # rubocop:disable GitlabSecurity/PublicSend def save_subrelations @@ -92,30 +91,6 @@ module Gitlab end end # rubocop:enable GitlabSecurity/PublicSend - - def log_invalid_subrelations - invalid_subrelations.flatten.each do |record| - Gitlab::Import::Logger.info( - message: '[Project/Group Import] Invalid subrelation', - importable_column_name => importable.id, - relation_key: relation_key, - error_messages: record.errors.full_messages.to_sentence - ) - - ImportFailure.create( - source: 'RelationObjectSaver#save!', - relation_key: relation_key, - exception_class: 'RecordInvalid', - exception_message: record.errors.full_messages.to_sentence, - correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, - importable_column_name => importable.id - ) - end - end - - def importable_column_name - @column_name ||= importable.class.reflect_on_association(:import_failures).foreign_key.to_sym - end end end end diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index 64ef3dd4830..d681f39f00b 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -90,6 +90,7 @@ module Gitlab def untar_with_options(archive:, dir:, options:) execute_cmd(%W(tar -#{options} #{archive} -C #{dir})) execute_cmd(%W(chmod -R #{UNTAR_MASK} #{dir})) + remove_symlinks(dir) end # rubocop:disable Gitlab/ModuleWithInstanceVariables @@ -120,6 +121,19 @@ module Gitlab FileUtils.copy_entry(source, destination) true end + + def remove_symlinks(dir) + ignore_file_names = %w[. ..] + + # Using File::FNM_DOTMATCH to also delete symlinks starting with "." + Dir.glob("#{dir}/**/*", File::FNM_DOTMATCH) + .reject { |f| ignore_file_names.include?(File.basename(f)) } + .each do |filepath| + FileUtils.rm(filepath) if File.lstat(filepath).symlink? + end + + true + end end end end diff --git a/lib/gitlab/import_export/config.rb b/lib/gitlab/import_export/config.rb index 83c4bc47349..e1a62e3b25a 100644 --- a/lib/gitlab/import_export/config.rb +++ b/lib/gitlab/import_export/config.rb @@ -10,6 +10,7 @@ module Gitlab @ee_hash = @hash.delete(:ee) || {} @hash[:tree] = normalize_tree(@hash[:tree]) + @hash[:import_only_tree] = normalize_tree(@hash[:import_only_tree] || {}) @ee_hash[:tree] = normalize_tree(@ee_hash[:tree] || {}) end @@ -51,7 +52,7 @@ module Gitlab end def parse_yaml - YAML.load_file(@config) + YAML.safe_load_file(@config, aliases: true, permitted_classes: [Symbol]) end end end diff --git a/lib/gitlab/import_export/file_importer.rb b/lib/gitlab/import_export/file_importer.rb index 1878b5b1a30..d2593289c23 100644 --- a/lib/gitlab/import_export/file_importer.rb +++ b/lib/gitlab/import_export/file_importer.rb @@ -8,7 +8,6 @@ module Gitlab ImporterError = Class.new(StandardError) MAX_RETRIES = 8 - IGNORED_FILENAMES = %w(. ..).freeze def self.import(*args, **kwargs) new(*args, **kwargs).import @@ -24,7 +23,7 @@ module Gitlab mkdir_p(@shared.export_path) mkdir_p(@shared.archive_path) - remove_symlinks + remove_symlinks(@shared.export_path) copy_archive wait_for_archived_file do @@ -36,7 +35,7 @@ module Gitlab false ensure remove_import_file - remove_symlinks + remove_symlinks(@shared.export_path) end private @@ -86,22 +85,10 @@ module Gitlab end end - def remove_symlinks - extracted_files.each do |path| - FileUtils.rm(path) if File.lstat(path).symlink? - end - - true - end - def remove_import_file FileUtils.rm_rf(@archive_file) end - def extracted_files - Dir.glob("#{@shared.export_path}/**/*", File::FNM_DOTMATCH).reject { |f| IGNORED_FILENAMES.include?(File.basename(f)) } - end - def validate_decompressed_archive_size raise ImporterError, _('Decompressed archive size validation failed.') unless size_validator.valid? end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index 5a78f2fb531..5453792e904 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -34,7 +34,6 @@ module Gitlab update_params! BulkInsertableAssociations.with_bulk_insert(enabled: bulk_insert_enabled) do - fix_ci_pipelines_not_sorted_on_legacy_project_json! create_relations! end end @@ -90,13 +89,23 @@ module Gitlab def save_relation_object(relation_object, relation_key, relation_definition, relation_index) if relation_object.new_record? - Gitlab::ImportExport::Base::RelationObjectSaver.new( + saver = Gitlab::ImportExport::Base::RelationObjectSaver.new( relation_object: relation_object, relation_key: relation_key, relation_definition: relation_definition, importable: @importable - ).execute + ) + + saver.execute + + log_invalid_subrelations(saver.invalid_subrelations, relation_key) else + if relation_object.invalid? + Gitlab::Import::Errors.merge_nested_errors(relation_object) + + raise(ActiveRecord::RecordInvalid, relation_object) + end + import_failure_service.with_retry(action: 'relation_object.save!', relation_key: relation_key, relation_index: relation_index) do relation_object.save! end @@ -113,7 +122,7 @@ module Gitlab @relations ||= @reader .attributes_finder - .find_relations_tree(importable_class_sym) + .find_relations_tree(importable_class_sym, include_import_only_tree: true) .deep_stringify_keys end @@ -126,9 +135,7 @@ module Gitlab modify_attributes - Gitlab::Timeless.timeless(@importable) do - @importable.save! - end + @importable.save!(touch: false) end def filter_attributes(params) @@ -265,15 +272,6 @@ module Gitlab } end - # Temporary fix for https://gitlab.com/gitlab-org/gitlab/-/issues/27883 when import from legacy project.json - # This should be removed once legacy JSON format is deprecated. - # Ndjson export file will fix the order during project export. - def fix_ci_pipelines_not_sorted_on_legacy_project_json! - return unless @relation_reader.legacy? - - @relation_reader.sort_ci_pipelines_by_id - end - # Enable logging of each top-level relation creation when Importing into a Group def log_relation_creation(importable, relation_key, relation_object) root_ancestor_group = importable.try(:root_ancestor) @@ -290,6 +288,32 @@ module Gitlab message: '[Project/Group Import] Created new object relation' ) end + + def log_invalid_subrelations(invalid_subrelations, relation_key) + invalid_subrelations.flatten.each do |record| + Gitlab::Import::Errors.merge_nested_errors(record) + + @shared.logger.info( + message: '[Project/Group Import] Invalid subrelation', + importable_column_name => @importable.id, + relation_key: relation_key, + error_messages: record.errors.full_messages.to_sentence + ) + + ::ImportFailure.create( + source: 'RelationTreeRestorer#save_relation_object', + relation_key: relation_key, + exception_class: 'ActiveRecord::RecordInvalid', + exception_message: record.errors.full_messages.to_sentence, + correlation_id_value: Labkit::Correlation::CorrelationId.current_or_new_id, + importable_column_name => @importable.id + ) + end + end + + def importable_column_name + @column_name ||= @importable.class.reflect_on_association(:import_failures).foreign_key.to_sym + end end end end diff --git a/lib/gitlab/import_export/json/legacy_reader.rb b/lib/gitlab/import_export/json/legacy_reader.rb deleted file mode 100644 index ee360020556..00000000000 --- a/lib/gitlab/import_export/json/legacy_reader.rb +++ /dev/null @@ -1,123 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Json - class LegacyReader - class File < LegacyReader - include Gitlab::Utils::StrongMemoize - - def initialize(path, relation_names:, allowed_path: nil) - @path = path - super( - relation_names: relation_names, - allowed_path: allowed_path) - end - - def exist? - ::File.exist?(@path) - end - - protected - - def tree_hash - strong_memoize(:tree_hash) do - read_hash - end - end - - def read_hash - Gitlab::Json.parse(::File.read(@path)) - rescue StandardError => e - Gitlab::ErrorTracking.log_exception(e) - raise Gitlab::ImportExport::Error, 'Incorrect JSON format' - end - end - - class Hash < LegacyReader - def initialize(tree_hash, relation_names:, allowed_path: nil) - @tree_hash = tree_hash - super( - relation_names: relation_names, - allowed_path: allowed_path) - end - - def exist? - @tree_hash.present? - end - - protected - - attr_reader :tree_hash - end - - def initialize(relation_names:, allowed_path:) - @relation_names = relation_names.map(&:to_s) - @consumed_relations = Set.new - - # This is legacy reader, to be used in transition - # period before `.ndjson`, - # we strong validate what is being readed - @allowed_path = allowed_path - end - - def exist? - raise NotImplementedError - end - - def legacy? - true - end - - def consume_attributes(importable_path) - unless importable_path == @allowed_path - raise ArgumentError, "Invalid #{importable_path} passed to `consume_attributes`. Use #{@allowed_path} instead." - end - - attributes - end - - def consume_relation(importable_path, key) - unless importable_path == @allowed_path - raise ArgumentError, "Invalid #{importable_name} passed to `consume_relation`. Use #{@allowed_path} instead." - end - - Enumerator.new do |documents| - next unless @consumed_relations.add?("#{importable_path}/#{key}") - - value = relations.delete(key) - next if value.nil? - - if value.is_a?(Array) - value.each.with_index do |item, idx| - documents << [item, idx] - end - else - documents << [value, 0] - end - end - end - - def sort_ci_pipelines_by_id - relations['ci_pipelines']&.sort_by! { |hash| hash['id'] } - end - - private - - attr_reader :relation_names, :allowed_path - - def tree_hash - raise NotImplementedError - end - - def attributes - @attributes ||= tree_hash.slice!(*relation_names) - end - - def relations - @relations ||= tree_hash.extract!(*relation_names) - end - end - end - end -end diff --git a/lib/gitlab/import_export/json/legacy_writer.rb b/lib/gitlab/import_export/json/legacy_writer.rb deleted file mode 100644 index e03ab9f7650..00000000000 --- a/lib/gitlab/import_export/json/legacy_writer.rb +++ /dev/null @@ -1,88 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ImportExport - module Json - class LegacyWriter - include Gitlab::ImportExport::CommandLineUtil - - attr_reader :path - - def initialize(path, allowed_path:) - @path = path - @keys = Set.new - - # This is legacy writer, to be used in transition - # period before `.ndjson`, - # we strong validate what is being written - @allowed_path = allowed_path - - mkdir_p(File.dirname(@path)) - file.write('{}') - end - - def close - @file&.close - @file = nil - end - - def write_attributes(exportable_path, hash) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - hash.each do |key, value| - write(key, value) - end - end - - def write_relation(exportable_path, key, value) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - write(key, value) - end - - def write_relation_array(exportable_path, key, items) - unless exportable_path == @allowed_path - raise ArgumentError, "Invalid #{exportable_path}" - end - - write(key, []) - - # rewind by two bytes, to overwrite ']}' - file.pos = file.size - 2 - - items.each_with_index do |item, idx| - file.write(',') if idx > 0 - file.write(item.to_json) - end - - file.write(']}') - end - - private - - def write(key, value) - raise ArgumentError, "key '#{key}' already written" if @keys.include?(key) - - # rewind by one byte, to overwrite '}' - file.pos = file.size - 1 - - file.write(',') if @keys.any? - file.write(key.to_json) - file.write(':') - file.write(value.to_json) - file.write('}') - - @keys.add(key) - end - - def file - @file ||= File.open(@path, "wb") - end - end - end - end -end diff --git a/lib/gitlab/import_export/json/ndjson_reader.rb b/lib/gitlab/import_export/json/ndjson_reader.rb index 510da61d3ab..3de56aacf18 100644 --- a/lib/gitlab/import_export/json/ndjson_reader.rb +++ b/lib/gitlab/import_export/json/ndjson_reader.rb @@ -17,14 +17,12 @@ module Gitlab Dir.exist?(@dir_path) end - # This can be removed once legacy_reader is deprecated. - def legacy? - false - end - def consume_attributes(importable_path) # This reads from `tree/project.json` path = file_path("#{importable_path}.json") + + raise Gitlab::ImportExport::Error, 'Invalid file' if !File.exist?(path) || File.symlink?(path) + data = File.read(path, MAX_JSON_DOCUMENT_SIZE) json_decode(data) end @@ -36,7 +34,7 @@ module Gitlab # This reads from `tree/project/merge_requests.ndjson` path = file_path(importable_path, "#{key}.ndjson") - next unless File.exist?(path) + next if !File.exist?(path) || File.symlink?(path) File.foreach(path, MAX_JSON_DOCUMENT_SIZE).with_index do |line, line_num| documents << [json_decode(line), line_num] diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 389ab8b4c97..9bb0770dc90 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -8,6 +8,8 @@ module Gitlab BATCH_SIZE = 100 + attr_reader :exported_objects_count + class Raw < String def to_json(*_args) to_s @@ -21,6 +23,7 @@ module Gitlab @relations_schema = relations_schema @json_writer = json_writer @logger = logger + @exported_objects_count = 0 end def execute @@ -40,21 +43,28 @@ module Gitlab relations_schema.merge(include: nil, preloads: nil, unsafe: true)) json_writer.write_attributes(exportable_path, attributes) + + increment_exported_objects_counter end - def serialize_relation(definition) + def serialize_relation(definition, options = {}) raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash) raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one? - key, options = definition.first + key, definition_options = definition.first record = exportable.public_send(key) # rubocop: disable GitlabSecurity/PublicSend + + if options[:batch_ids] + record = record.where(record.model.primary_key => Array.wrap(options[:batch_ids]).map(&:to_i)) + end + if record.is_a?(ActiveRecord::Relation) - serialize_many_relations(key, record, options) + serialize_many_relations(key, record, definition_options) elsif record.respond_to?(:each) # this is to support `project_members` that return an Array - serialize_many_each(key, record, options) + serialize_many_each(key, record, definition_options) else - serialize_single_relation(key, record, options) + serialize_single_relation(key, record, definition_options) end end @@ -76,6 +86,8 @@ module Gitlab items << exportable_json_record(record, options, key) + increment_exported_objects_counter + after_read_callback(record) end end @@ -175,6 +187,8 @@ module Gitlab enumerator = Enumerator.new do |items| records.each do |record| items << exportable_json_record(record, options, key) + + increment_exported_objects_counter end end @@ -187,6 +201,8 @@ module Gitlab json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) + + increment_exported_objects_counter end def includes @@ -263,6 +279,10 @@ module Gitlab message += ". Number of records to export: #{size}" if size logger.info(message: message, **log_base_data) end + + def increment_exported_objects_counter + @exported_objects_count += 1 + end end end end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index d97ffee8698..36a3c73271b 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -89,13 +89,15 @@ tree: - :milestone - :resource_state_events - :external_pull_requests + - commit_notes: + - :author + - events: + - :push_event_payload - ci_pipelines: - - notes: - - :author - - events: - - :push_event_payload - stages: - - :statuses + - :builds + - :generic_commit_statuses + - :bridges - :external_pull_request - :merge_request - :pipeline_metadata @@ -119,9 +121,25 @@ tree: - label: - :priorities - :service_desk_setting + - :design_management_repository group_members: - :user +# Used to support old exports that were exported before the removal/rename of the associations +# +# For example, statuses of ci_pipelines are no longer exported, and instead, statuses are +# exported as builds, generic_commit_statuses, and bridges. So in order to allow statuses +# to be still imported, it is added to the list below. +import_only_tree: + project: + - ci_pipelines: + - notes: + - :author + - events: + - :push_event_payload + - stages: + - :statuses + # Only include the following attributes for the models specified. included_attributes: user: @@ -529,7 +547,7 @@ included_attributes: - :source_sha - :target_sha external_pull_requests: *external_pull_request_definition - statuses: + statuses: &statuses_definition - :project_id - :status - :finished_at @@ -562,6 +580,9 @@ included_attributes: - :scheduled_at - :scheduling_type - :ci_stage + builds: *statuses_definition + generic_commit_statuses: *statuses_definition + bridges: *statuses_definition ci_pipelines: - :ref - :sha @@ -596,6 +617,7 @@ included_attributes: - :project_id - :created_at - :updated_at + # - :statuses # old exports use statuses instead of builds, generic_commit_statuses and bridges actions: - :event design: &design_definition @@ -896,7 +918,7 @@ excluded_attributes: merge_requests: *merge_request_excluded_definition award_emoji: - :awardable_id - statuses: + statuses: &statuses_excluded_definition - :trace - :token - :token_encrypted @@ -918,6 +940,9 @@ excluded_attributes: - :processed - :id_convert_to_bigint - :stage_id_convert_to_bigint + builds: *statuses_excluded_definition + generic_commit_statuses: *statuses_excluded_definition + bridges: *statuses_excluded_definition sentry_issue: - :issue_id push_event_payload: @@ -955,6 +980,9 @@ excluded_attributes: notes: - :noteable_id - :review_id + commit_notes: + - :noteable_id + - :review_id label_links: - :label_id - :target_id @@ -1079,6 +1107,8 @@ methods: - :squash_option notes: - :type + commit_notes: + - :type labels: - :type label: @@ -1100,8 +1130,6 @@ methods: - :type lists: - :list_type - ci_pipelines: - - :notes issues: - :state @@ -1111,10 +1139,11 @@ methods: preloads: issues: project: :route - statuses: - # TODO: We cannot preload tags, as they are not part of `GenericCommitStatus` - # tags: # needed by tag_list - project: # deprecated: needed by coverage_regex of Ci::Build + builds: + metadata: + project: + bridges: + metadata: merge_requests: source_project: :route # needed by source_branch_sha and diff_head_sha target_project: :route # needed by target_branch_sha @@ -1167,6 +1196,9 @@ ee: - :milestone - lists: - :milestone + - approval_rules: + - :approval_project_rules_protected_branches + - :approval_project_rules_users included_attributes: issuable_sla: @@ -1232,9 +1264,30 @@ ee: - :description iterations_cadence: - :title + approval_rules: + - :approvals_required + - :name + - :rule_type + - :scanners + - :vulnerabilities_allowed + - :severity_levels + - :report_type + - :vulnerability_states + - :orchestration_policy_idx + - :applies_to_all_protected_branches + approval_project_rules_protected_branches: + - :protected_branch + approval_project_rules_users: + - :user_id excluded_attributes: project: - :vulnerability_hooks_integrations + approval_rules: + - :created_at + - :updated_at + methods: + approval_project_rules_protected_branches: + - :branch_name preloads: issues: epic: diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index 50a67a746f8..ac28ae6bfe0 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -60,10 +60,11 @@ module Gitlab def prepare_attributes attributes.dup.tap do |atts| - atts.delete('group') unless epic? || iteration? + atts.delete('group') unless group_level_object? if label? atts['type'] = 'ProjectLabel' # Always create project labels + atts.delete('group_id') elsif milestone? if atts['group_id'] # Transform new group milestones into project ones atts['iid'] = nil @@ -141,10 +142,6 @@ module Gitlab klass == MergeRequestDiffCommit end - def iteration? - klass == Iteration - end - # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -163,7 +160,11 @@ module Gitlab end def group_relation_without_group? - (epic? || iteration?) && group.nil? + group_level_object? && group.nil? + end + + def group_level_object? + epic? end end end diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 4134c428500..5d7e3ea9ed7 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -5,6 +5,7 @@ module Gitlab module Project class RelationFactory < Base::RelationFactory OVERRIDES = { snippets: :project_snippets, + commit_notes: 'Note', ci_pipelines: 'Ci::Pipeline', pipelines: 'Ci::Pipeline', stages: 'Ci::Stage', @@ -12,6 +13,7 @@ module Gitlab triggers: 'Ci::Trigger', pipeline_schedules: 'Ci::PipelineSchedule', builds: 'Ci::Build', + bridges: 'Ci::Bridge', runners: 'Ci::Runner', pipeline_metadata: 'Ci::PipelineMetadata', hooks: 'ProjectHook', @@ -20,6 +22,7 @@ module Gitlab create_access_levels: 'ProtectedTag::CreateAccessLevel', design: 'DesignManagement::Design', designs: 'DesignManagement::Design', + design_management_repository: 'DesignManagement::Repository', design_versions: 'DesignManagement::Version', actions: 'DesignManagement::Action', labels: :project_labels, @@ -37,7 +40,7 @@ module Gitlab committer: 'MergeRequest::DiffCommitUser', merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze - BUILD_MODELS = %i[Ci::Build commit_status].freeze + BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze GROUP_REFERENCES = %w[group_id].freeze @@ -83,13 +86,14 @@ module Gitlab def setup_models case @relation_name when :merge_request_diff_files then setup_diff - when :notes then setup_note + when :notes, :Note then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build when :issues then setup_issue when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level + when :ApprovalProjectRulesProtectedBranch then setup_merge_approval_protected_branch when :releases then setup_release end @@ -142,9 +146,22 @@ module Gitlab def setup_pipeline @relation_hash.fetch('stages', []).each do |stage| + # old export files have statuses stage.statuses.each do |status| status.pipeline = imported_object end + + stage.builds.each do |status| + status.pipeline = imported_object + end + + stage.bridges.each do |status| + status.pipeline = imported_object + end + + stage.generic_commit_statuses.each do |status| + status.pipeline = imported_object + end end end @@ -180,6 +197,13 @@ module Gitlab root_ancestor.max_member_access_for_user(@user) == Gitlab::Access::OWNER end + def setup_merge_approval_protected_branch + source_branch_name = @relation_hash.delete('branch_name') + target_branch = @importable.protected_branches.find_by(name: source_branch_name) + + @relation_hash['protected_branch'] = target_branch + end + def compute_relative_position return unless max_relative_position diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb index 47196db6f8a..b5247754199 100644 --- a/lib/gitlab/import_export/project/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -5,10 +5,14 @@ module Gitlab module Project class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze private + def group_models + GROUP_MODELS + end + def bulk_insert_enabled true end @@ -19,9 +23,11 @@ module Gitlab end def relation_invalid_for_importable?(relation_object) - GROUP_MODELS.include?(relation_object.class) && relation_object.group_id + group_models.include?(relation_object.class) && relation_object.group_id end end end end end + +Gitlab::ImportExport::Project::RelationTreeRestorer.prepend_mod diff --git a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb index 034122a9f14..639f34980ff 100644 --- a/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/sample/relation_tree_restorer.rb @@ -18,8 +18,6 @@ module Gitlab end def dates - return [] if @relation_reader.legacy? - RelationFactory::DATE_MODELS.flat_map do |tag| @relation_reader.consume_relation(@importable_path, tag, mark_as_consumed: false).map do |model| model.first['due_date'] diff --git a/lib/gitlab/import_export/project/tree_restorer.rb b/lib/gitlab/import_export/project/tree_restorer.rb index 47f82a901b7..e791424875a 100644 --- a/lib/gitlab/import_export/project/tree_restorer.rb +++ b/lib/gitlab/import_export/project/tree_restorer.rb @@ -17,7 +17,7 @@ module Gitlab end def restore - unless relation_reader + unless relation_reader.exist? raise Gitlab::ImportExport::Error, 'invalid import format' end @@ -47,28 +47,11 @@ module Gitlab private def relation_reader - strong_memoize(:relation_reader) do - [ndjson_relation_reader, legacy_relation_reader] - .compact.find(&:exist?) - end - end - - def ndjson_relation_reader - return unless Feature.enabled?(:project_import_ndjson, project.namespace) - - ImportExport::Json::NdjsonReader.new( + @relation_reader ||= ImportExport::Json::NdjsonReader.new( File.join(shared.export_path, 'tree') ) end - def legacy_relation_reader - ImportExport::Json::LegacyReader::File.new( - File.join(shared.export_path, 'project.json'), - relation_names: reader.project_relation_names, - allowed_path: importable_path - ) - end - def relation_tree_restorer @relation_tree_restorer ||= relation_tree_restorer_class.new( user: @user, diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 05b96f7e8ce..fd5fa73764e 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -81,13 +81,10 @@ module Gitlab end def json_writer - @json_writer ||= if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) - full_path = File.join(@shared.export_path, 'tree') - Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) - else - full_path = File.join(@shared.export_path, ImportExport.project_filename) - Gitlab::ImportExport::Json::LegacyWriter.new(full_path, allowed_path: 'project') - end + @json_writer ||= begin + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::Json::NdjsonWriter.new(full_path) + end end end end diff --git a/lib/gitlab/import_sources.rb b/lib/gitlab/import_sources.rb index a94ea6f595b..77b0df765c4 100644 --- a/lib/gitlab/import_sources.rb +++ b/lib/gitlab/import_sources.rb @@ -9,18 +9,15 @@ module Gitlab module ImportSources ImportSource = Struct.new(:name, :title, :importer) - # We exclude `bare_repository` here as it has no import class associated IMPORT_TABLE = [ ImportSource.new('github', 'GitHub', Gitlab::GithubImport::ParallelImporter), ImportSource.new('bitbucket', 'Bitbucket Cloud', Gitlab::BitbucketImport::Importer), ImportSource.new('bitbucket_server', 'Bitbucket Server', Gitlab::BitbucketServerImport::Importer), - ImportSource.new('gitlab', 'GitLab.com', Gitlab::GitlabImport::Importer), ImportSource.new('fogbugz', 'FogBugz', Gitlab::FogbugzImport::Importer), ImportSource.new('git', 'Repository by URL', nil), ImportSource.new('gitlab_project', 'GitLab export', Gitlab::ImportExport::Importer), ImportSource.new('gitea', 'Gitea', Gitlab::LegacyGithubImport::Importer), - ImportSource.new('manifest', 'Manifest file', nil), - ImportSource.new('phabricator', 'Phabricator', Gitlab::PhabricatorImport::Importer) + ImportSource.new('manifest', 'Manifest file', nil) ].freeze class << self diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb deleted file mode 100644 index d34c19bc9fc..00000000000 --- a/lib/gitlab/incoming_email.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module IncomingEmail - class << self - include Gitlab::Email::Common - - def config - incoming_email_config - end - - def key_from_address(address, wildcard_address: nil) - wildcard_address ||= config.address - regex = address_regex(wildcard_address) - return unless regex - - match = address.match(regex) - return unless match - - match[1] - end - - private - - def address_regex(wildcard_address) - return unless wildcard_address - - regex = Regexp.escape(wildcard_address) - regex = regex.sub(Regexp.escape(WILDCARD_PLACEHOLDER), '(.+)') - Regexp.new(/\A<?#{regex}>?\z/).freeze - end - end - end -end diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index a664656c467..590153ad9cd 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -19,8 +19,8 @@ module Gitlab end << ActionCable ).freeze - # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). - QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze + # Milliseconds represented in seconds + QUERY_TIME_BUCKETS = [0.1, 0.25, 0.5].freeze class << self include ::Gitlab::Instrumentation::RedisPayload diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index de24132a28e..00a7387afe2 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -118,6 +118,14 @@ module Gitlab @exception_counter.increment({ storage: storage_key, exception: ex.class.to_s }) end + def instance_count_cluster_redirection(ex) + # This metric is meant to give a client side view of how often are commands + # redirected to the right node, especially during resharding.. + # This metric can be used for Redis alerting and service health monitoring. + @redirection_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_redirections_total, 'Client side Redis Cluster redirection count, per Redis node, per slot') + @redirection_counter.increment(decompose_redirection_message(ex.message).merge({ storage: storage_key })) + end + def instance_observe_duration(duration) @request_latency_histogram ||= Gitlab::Metrics.histogram( :gitlab_redis_client_requests_duration_seconds, @@ -129,6 +137,10 @@ module Gitlab @request_latency_histogram.observe({ storage: storage_key }, duration) end + def log_exception(ex) + ::Gitlab::ErrorTracking.log_exception(ex, storage: storage_key) + end + private def request_count_key @@ -162,6 +174,11 @@ module Gitlab def build_key(namespace) "#{storage_key}_#{namespace}" end + + def decompose_redirection_message(err_msg) + redirection_type, _, target_node_key = err_msg.split + { redirection_type: redirection_type, target_node_key: target_node_key } + end end end end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 35dd7cbfeb8..b3fbe30e583 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -3,7 +3,7 @@ module Gitlab module Instrumentation module RedisInterceptor - APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze + APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax command xread xreadgroup].freeze def call(command) instrument_call([command]) do @@ -40,7 +40,13 @@ module Gitlab yield rescue ::Redis::BaseError => ex - instrumentation_class.instance_count_exception(ex) + if ex.message.start_with?('MOVED', 'ASK') + instrumentation_class.instance_count_cluster_redirection(ex) + else + instrumentation_class.instance_count_exception(ex) + end + + instrumentation_class.log_exception(ex) raise ex ensure duration = Gitlab::Metrics::System.monotonic_time - start diff --git a/lib/gitlab/instrumentation/zoekt.rb b/lib/gitlab/instrumentation/zoekt.rb new file mode 100644 index 00000000000..cd9b15bcee8 --- /dev/null +++ b/lib/gitlab/instrumentation/zoekt.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Instrumentation + class Zoekt + ZOEKT_REQUEST_COUNT = :zoekt_request_count + ZOEKT_CALL_DURATION = :zoekt_call_duration + ZOEKT_CALL_DETAILS = :zoekt_call_details + + class << self + def get_request_count + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] || 0 + end + + def increment_request_count + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_REQUEST_COUNT] += 1 + end + + def detail_store + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DETAILS] ||= [] + end + + def query_time + query_time = ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] || 0 + query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) + end + + def add_duration(duration) + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] ||= 0 + ::Gitlab::SafeRequestStore[ZOEKT_CALL_DURATION] += duration + end + + def add_call_details(duration:, method:, path:, params: nil, body: nil) + return unless Gitlab::PerformanceBar.enabled_for_request? + + detail_store << { + method: method, + path: path, + params: params, + body: body, + duration: duration, + backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) + } + end + end + end + end +end diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 15a760fada0..2a3c4db5ffa 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -6,16 +6,8 @@ module Gitlab DURATION_PRECISION = 6 # microseconds - def init_instrumentation_data(request_ip: nil) - # Set `request_start_time` only if this is request - # This is done, as `request_start_time` imply `request_deadline` - if request_ip - Gitlab::RequestContext.instance.client_ip = request_ip - Gitlab::RequestContext.instance.request_start_time = Gitlab::Metrics::System.real_time - end - - Gitlab::RequestContext.instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time - Gitlab::RequestContext.instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations + def init_instrumentation_data + Gitlab::RequestContext.start_thread_context end def add_instrumentation_data(payload) @@ -23,6 +15,7 @@ module Gitlab instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) + instrument_zoekt(payload) instrument_throttle(payload) instrument_active_record(payload) instrument_external_http(payload) @@ -72,6 +65,17 @@ module Gitlab payload[:elasticsearch_timed_out_count] = Gitlab::Instrumentation::ElasticsearchTransport.get_timed_out_count end + def instrument_zoekt(payload) + # Zoekt integration is only available in EE but instrumentation + # only depends on the Gem which is also available in FOSS. + zoekt_calls = Gitlab::Instrumentation::Zoekt.get_request_count + + return if zoekt_calls == 0 + + payload[:zoekt_calls] = zoekt_calls + payload[:zoekt_duration_s] = Gitlab::Instrumentation::Zoekt.query_time + end + def instrument_external_http(payload) external_http_count = Gitlab::Metrics::Subscribers::ExternalHttp.request_count diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb index 448ac4c2ae0..22b0d5c4fb2 100644 --- a/lib/gitlab/issuable/clone/copy_resource_events_service.rb +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -69,9 +69,9 @@ module Gitlab def copy_events(table_name, events_to_copy) events_to_copy.find_in_batches do |batch| - events = batch.map do |event| + events = batch.filter_map do |event| yield(event) - end.compact + end ApplicationRecord.legacy_bulk_insert(table_name, events) # rubocop:disable Gitlab/BulkInsert end diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index 36346564b39..f1f6cc55a2b 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -38,10 +38,10 @@ module Gitlab def rebalance_in_progress? is_running = case rebalanced_container_type when NAMESPACE - namespace_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{NAMESPACE}/").second.to_i }.compact + namespace_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{NAMESPACE}/").second.to_i } namespace_ids.include?(root_namespace.id) when PROJECT - project_ids = self.class.current_rebalancing_containers.map { |string| string.split("#{PROJECT}/").second.to_i }.compact + project_ids = self.class.current_rebalancing_containers.filter_map { |string| string.split("#{PROJECT}/").second.to_i } project_ids.include?(projects.take.id) # rubocop:disable CodeReuse/ActiveRecord else false diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 7b031c26b72..458f7c3f470 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -48,7 +48,7 @@ module Gitlab end def schedule_issue_import_workers(issues) - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) issues.each do |jira_issue| # Technically it's possible that the same work is performed multiple @@ -71,7 +71,7 @@ module Gitlab job_waiter.jobs_remaining += 1 - next_iid = Issue.with_project_iid_supply(project, &:next_value) + next_iid = Issue.with_namespace_iid_supply(project.project_namespace, &:next_value) # Mark the issue as imported immediately so we don't end up # importing it multiple times within same import. diff --git a/lib/gitlab/jira_import/metadata_collector.rb b/lib/gitlab/jira_import/metadata_collector.rb index 4551f38ba98..090b95ac14a 100644 --- a/lib/gitlab/jira_import/metadata_collector.rb +++ b/lib/gitlab/jira_import/metadata_collector.rb @@ -45,7 +45,7 @@ module Gitlab def add_versions return if fields['fixVersions'].blank? || !fields['fixVersions'].is_a?(Array) - versions = fields['fixVersions'].map { |version| version['name'] }.compact.join(', ') + versions = fields['fixVersions'].filter_map { |version| version['name'] }.join(', ') metadata << "- Fix versions: #{versions}" end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index bdfbe2041cd..e515d00f8d8 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -167,7 +167,7 @@ module Gitlab # @return [Boolean, String, Array, Hash, Object] # @raise [JSON::ParserError] def handle_legacy_mode!(data) - return data unless Feature.feature_flags_available? + return data unless Feature::FlipperFeature.table_exists? return data unless Feature.enabled?(:json_wrapper_legacy_mode) raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } diff --git a/lib/gitlab/json_cache.rb b/lib/gitlab/json_cache.rb index d2916a01809..0087c2accc3 100644 --- a/lib/gitlab/json_cache.rb +++ b/lib/gitlab/json_cache.rb @@ -112,7 +112,7 @@ module Gitlab end def parse_entries(values, klass) - values.map { |value| parse_entry(value, klass) }.compact + values.filter_map { |value| parse_entry(value, klass) } end end end diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 768810d5545..43546d04087 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -8,7 +8,8 @@ module Gitlab STUB_CLASSES = { agent_tracker: Gitlab::Agent::AgentTracker::Rpc::AgentTracker::Stub, - configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub + configuration_project: Gitlab::Agent::ConfigurationProject::Rpc::ConfigurationProject::Stub, + notifications: Gitlab::Agent::Notifications::Rpc::Notifications::Stub }.freeze ConfigurationError = Class.new(StandardError) @@ -39,6 +40,18 @@ module Gitlab .to_a end + def send_git_push_event(project:) + request = Gitlab::Agent::Notifications::Rpc::GitPushEventRequest.new( + project: Gitlab::Agent::Notifications::Rpc::Project.new( + id: project.id, + full_path: project.full_path + ) + ) + + stub_for(:notifications) + .git_push_event(request, metadata: metadata) + end + private def stub_for(service) diff --git a/lib/gitlab/kas/user_access.rb b/lib/gitlab/kas/user_access.rb new file mode 100644 index 00000000000..65ae399d826 --- /dev/null +++ b/lib/gitlab/kas/user_access.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Kas + # The name of the cookie that will be used for the KAS cookie + COOKIE_KEY = '_gitlab_kas' + DEFAULT_ENCRYPTED_COOKIE_CIPHER = 'aes-256-gcm' + + class UserAccess + class << self + def enabled? + ::Gitlab::Kas.enabled? && ::Feature.enabled?(:kas_user_access) + end + + def enabled_for?(agent) + enabled? && ::Feature.enabled?(:kas_user_access_project, agent.project) + end + + def encrypt_public_session_id(data) + encryptor.encrypt_and_sign(data.to_json, purpose: public_session_id_purpose) + end + + def decrypt_public_session_id(data) + decrypted = encryptor.decrypt_and_verify(data, purpose: public_session_id_purpose) + ::Gitlab::Json.parse(decrypted) + end + + def valid_authenticity_token?(session, masked_authenticity_token) + # rubocop:disable GitlabSecurity/PublicSend + ActionController::Base.new.send(:valid_authenticity_token?, session, masked_authenticity_token) + # rubocop:enable GitlabSecurity/PublicSend + end + + def cookie_data(public_session_id) + uri = URI(::Gitlab::Kas.tunnel_url) + + cookie = { + value: encrypt_public_session_id(public_session_id), + expires: 1.day, + httponly: true, + path: uri.path.presence || '/', + secure: Gitlab.config.gitlab.https + } + # Only set domain attribute if KAS is on a subdomain. + # When on the same domain, we can omit the attribute. + gitlab_host = Gitlab.config.gitlab.host + cookie[:domain] = gitlab_host if uri.host.end_with?(".#{gitlab_host}") + + cookie + end + + private + + def encryptor + action_dispatch_config = Gitlab::Application.config.action_dispatch + serializer = ActiveSupport::MessageEncryptor::NullSerializer + key_generator = ::Gitlab::Application.key_generator + + cipher = action_dispatch_config.encrypted_cookie_cipher || DEFAULT_ENCRYPTED_COOKIE_CIPHER + salt = action_dispatch_config.authenticated_encrypted_cookie_salt + key_len = ActiveSupport::MessageEncryptor.key_len(cipher) + secret = key_generator.generate_key(salt, key_len) + + ActiveSupport::MessageEncryptor.new(secret, cipher: cipher, serializer: serializer) + end + + def public_session_id_purpose + "kas.user_public_session_id" + end + end + end + end +end diff --git a/lib/gitlab/kubernetes/helm/api.rb b/lib/gitlab/kubernetes/helm/api.rb deleted file mode 100644 index ceda18442d6..00000000000 --- a/lib/gitlab/kubernetes/helm/api.rb +++ /dev/null @@ -1,126 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class API - def initialize(kubeclient) - @kubeclient = kubeclient - @namespace = Gitlab::Kubernetes::Namespace.new( - Gitlab::Kubernetes::Helm::NAMESPACE, - kubeclient, - labels: Gitlab::Kubernetes::Helm::NAMESPACE_LABELS - ) - end - - def install(command) - namespace.ensure_exists! - - create_service_account(command) - create_cluster_role_binding(command) - create_config_map(command) - - delete_pod!(command.pod_name) - kubeclient.create_pod(command.pod_resource) - end - - alias_method :update, :install - - def uninstall(command) - namespace.ensure_exists! - create_config_map(command) - - delete_pod!(command.pod_name) - kubeclient.create_pod(command.pod_resource) - end - - ## - # Returns Pod phase - # - # https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase - # - # values: "Pending", "Running", "Succeeded", "Failed", "Unknown" - # - def status(pod_name) - kubeclient.get_pod(pod_name, namespace.name).status.phase - end - - def log(pod_name) - kubeclient.get_pod_log(pod_name, namespace.name).body - end - - def delete_pod!(pod_name) - kubeclient.delete_pod(pod_name, namespace.name) - rescue ::Kubeclient::ResourceNotFoundError - # no-op - end - - def get_config_map(config_map_name) - namespace.ensure_exists! - - kubeclient.get_config_map(config_map_name, namespace.name) - end - - private - - attr_reader :kubeclient, :namespace - - def create_config_map(command) - command.config_map_resource.tap do |config_map_resource| - break unless config_map_resource - - if config_map_exists?(config_map_resource) - kubeclient.update_config_map(config_map_resource) - else - kubeclient.create_config_map(config_map_resource) - end - end - end - - def update_config_map(command) - command.config_map_resource.tap do |config_map_resource| - kubeclient.update_config_map(config_map_resource) - end - end - - def create_service_account(command) - command.service_account_resource.tap do |service_account_resource| - break unless service_account_resource - - if service_account_exists?(service_account_resource) - kubeclient.update_service_account(service_account_resource) - else - kubeclient.create_service_account(service_account_resource) - end - end - end - - def create_cluster_role_binding(command) - command.cluster_role_binding_resource.tap do |cluster_role_binding_resource| - break unless cluster_role_binding_resource - - kubeclient.update_cluster_role_binding(cluster_role_binding_resource) - end - end - - def config_map_exists?(resource) - kubeclient.get_config_map(resource.metadata.name, resource.metadata.namespace) - rescue ::Kubeclient::ResourceNotFoundError - false - end - - def service_account_exists?(resource) - kubeclient.get_service_account(resource.metadata.name, resource.metadata.namespace) - rescue ::Kubeclient::ResourceNotFoundError - false - end - - def cluster_role_binding_exists?(resource) - kubeclient.get_cluster_role_binding(resource.metadata.name) - rescue ::Kubeclient::ResourceNotFoundError - false - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/pod.rb b/lib/gitlab/kubernetes/helm/pod.rb deleted file mode 100644 index 9d0207e6b1f..00000000000 --- a/lib/gitlab/kubernetes/helm/pod.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - class Pod - def initialize(command, namespace_name, service_account_name: nil) - @command = command - @namespace_name = namespace_name - @service_account_name = service_account_name - end - - def generate - spec = { containers: [container_specification], restartPolicy: 'Never' } - - spec[:volumes] = volumes_specification - spec[:containers][0][:volumeMounts] = volume_mounts_specification - spec[:serviceAccountName] = service_account_name if service_account_name - - ::Kubeclient::Resource.new(metadata: metadata, spec: spec) - end - - private - - attr_reader :command, :namespace_name, :service_account_name - - def container_specification - { - name: 'helm', - image: "registry.gitlab.com/gitlab-org/cluster-integration/helm-install-image/releases/#{command.class::HELM_VERSION}-kube-#{Gitlab::Kubernetes::Helm::KUBECTL_VERSION}-alpine-3.12", - env: generate_pod_env(command), - command: %w(/bin/sh), - args: %w(-c $(COMMAND_SCRIPT)) - } - end - - def labels - { - 'gitlab.org/action': 'install', - 'gitlab.org/application': command.name - } - end - - def metadata - { - name: command.pod_name, - namespace: namespace_name, - labels: labels - } - end - - def generate_pod_env(command) - command.env.merge( - HELM_VERSION: command.class::HELM_VERSION, - COMMAND_SCRIPT: command.generate_script - ).map { |key, value| { name: key, value: value } } - end - - def volumes_specification - [ - { - name: 'configuration-volume', - configMap: { - name: "values-content-configuration-#{command.name}", - items: command.file_names.map { |name| { key: name, path: name } } - } - } - ] - end - - def volume_mounts_specification - [ - { - name: 'configuration-volume', - mountPath: "/data/helm/#{command.name}/config" - } - ] - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/base_command.rb b/lib/gitlab/kubernetes/helm/v2/base_command.rb deleted file mode 100644 index 26c77b2149e..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/base_command.rb +++ /dev/null @@ -1,93 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - class BaseCommand - attr_reader :name, :files - - HELM_VERSION = '2.17.0' - - def initialize(rbac:, name:, files:) - @rbac = rbac - @name = name - @files = files - end - - def env - { TILLER_NAMESPACE: namespace } - end - - def rbac? - @rbac - end - - def pod_resource - pod_service_account_name = rbac? ? service_account_name : nil - - Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate - end - - def generate_script - <<~HEREDOC - set -xeo pipefail - HEREDOC - end - - def pod_name - "install-#{name}" - end - - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, files).generate - end - - def service_account_resource - return unless rbac? - - Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate - end - - def cluster_role_binding_resource - return unless rbac? - - subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] - - Gitlab::Kubernetes::ClusterRoleBinding.new( - cluster_role_binding_name, - cluster_role_name, - subjects - ).generate - end - - def file_names - files.keys - end - - private - - def files_dir - "/data/helm/#{name}/config" - end - - def namespace - Gitlab::Kubernetes::Helm::NAMESPACE - end - - def service_account_name - Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT - end - - def cluster_role_binding_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING - end - - def cluster_role_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/certificate.rb b/lib/gitlab/kubernetes/helm/v2/certificate.rb deleted file mode 100644 index 17ea2eb5188..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/certificate.rb +++ /dev/null @@ -1,75 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module Kubernetes - module Helm - module V2 - class Certificate - INFINITE_EXPIRY = 1000.years - SHORT_EXPIRY = 30.minutes - - attr_reader :key, :cert - - def key_string - @key.to_s - end - - def cert_string - @cert.to_pem - end - - def self.from_strings(key_string, cert_string) - key = OpenSSL::PKey::RSA.new(key_string) - cert = OpenSSL::X509::Certificate.new(cert_string) - new(key, cert) - end - - def self.generate_root - _issue(signed_by: nil, expires_in: INFINITE_EXPIRY, certificate_authority: true) - end - - def issue(expires_in: SHORT_EXPIRY) - self.class._issue(signed_by: self, expires_in: expires_in, certificate_authority: false) - end - - private - - def self._issue(signed_by:, expires_in:, certificate_authority:) - key = OpenSSL::PKey::RSA.new(4096) - public_key = key.public_key - - subject = OpenSSL::X509::Name.parse("/C=US") - - cert = OpenSSL::X509::Certificate.new - cert.subject = subject - - cert.issuer = signed_by&.cert&.subject || subject - - cert.not_before = Time.now.utc - cert.not_after = expires_in.from_now.utc - cert.public_key = public_key - cert.serial = 0x0 - cert.version = 2 - - if certificate_authority - extension_factory = OpenSSL::X509::ExtensionFactory.new - extension_factory.subject_certificate = cert - extension_factory.issuer_certificate = cert - cert.add_extension(extension_factory.create_extension('subjectKeyIdentifier', 'hash')) - cert.add_extension(extension_factory.create_extension('basicConstraints', 'CA:TRUE', true)) - cert.add_extension(extension_factory.create_extension('keyUsage', 'cRLSign,keyCertSign', true)) - end - - cert.sign(signed_by&.key || key, OpenSSL::Digest.new('SHA256')) - - new(key, cert) - end - - def initialize(key, cert) - @key = key - @cert = cert - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/client_command.rb b/lib/gitlab/kubernetes/helm/v2/client_command.rb deleted file mode 100644 index 8b15af9aeea..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/client_command.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - module ClientCommand - def init_command - <<~SHELL.chomp - export HELM_HOST="localhost:44134" - tiller -listen ${HELM_HOST} -alsologtostderr & - helm init --client-only - SHELL - end - - def repository_command - ['helm', 'repo', 'add', name, repository].shelljoin if repository - end - - private - - def repository_update_command - 'helm repo update' - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/delete_command.rb b/lib/gitlab/kubernetes/helm/v2/delete_command.rb deleted file mode 100644 index 4d52fc1398f..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/delete_command.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - class DeleteCommand < BaseCommand - include ClientCommand - - attr_reader :predelete, :postdelete - - def initialize(predelete: nil, postdelete: nil, **args) - super(**args) - @predelete = predelete - @postdelete = postdelete - end - - def generate_script - super + [ - init_command, - predelete, - delete_command, - postdelete - ].compact.join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - def delete_command - ['helm', 'delete', '--purge', name].shelljoin - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/init_command.rb b/lib/gitlab/kubernetes/helm/v2/init_command.rb deleted file mode 100644 index f8b52feb5b6..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/init_command.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - class InitCommand < BaseCommand - def generate_script - super + [ - init_helm_command - ].join("\n") - end - - private - - def init_helm_command - command = %w[helm init] + init_command_flags - - command.shelljoin - end - - def init_command_flags - tls_flags + optional_service_account_flag - end - - def tls_flags - [ - '--tiller-tls', - '--tiller-tls-verify', - '--tls-ca-cert', "#{files_dir}/ca.pem", - '--tiller-tls-cert', "#{files_dir}/cert.pem", - '--tiller-tls-key', "#{files_dir}/key.pem" - ] - end - - def optional_service_account_flag - return [] unless rbac? - - ['--service-account', service_account_name] - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/install_command.rb b/lib/gitlab/kubernetes/helm/v2/install_command.rb deleted file mode 100644 index c50db6bf177..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/install_command.rb +++ /dev/null @@ -1,87 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - class InstallCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository, :preinstall, :postinstall - attr_accessor :version - - def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) - super(**args) - @chart = chart - @version = version - @repository = repository - @preinstall = preinstall - @postinstall = postinstall - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - preinstall, - install_command, - postinstall - ].compact.join("\n") - end - - private - - # Uses `helm upgrade --install` which means we can use this for both - # installation and uprade of applications - def install_command - command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def install_flag - ['--install'] - end - - def reset_values_flag - ['--reset-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def rbac_create_flag - if rbac? - %w[--set rbac.create=true,rbac.enabled=true] - else - %w[--set rbac.create=false,rbac.enabled=false] - end - end - - def optional_version_flag - return [] unless version - - ['--version', version] - end - - def rollback_support_flag - ['--atomic', '--cleanup-on-fail'] - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/patch_command.rb b/lib/gitlab/kubernetes/helm/v2/patch_command.rb deleted file mode 100644 index 40e56771e47..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/patch_command.rb +++ /dev/null @@ -1,67 +0,0 @@ -# frozen_string_literal: true - -# PatchCommand is for updating values in installed charts without overwriting -# existing values. -module Gitlab - module Kubernetes - module Helm - module V2 - class PatchCommand < BaseCommand - include ClientCommand - - attr_reader :chart, :repository - attr_accessor :version - - def initialize(chart:, version:, repository: nil, **args) - super(**args) - - # version is mandatory to prevent chart mismatches - # we do not want our values interpreted in the context of the wrong version - raise ArgumentError, 'version is required' if version.blank? - - @chart = chart - @version = version - @repository = repository - end - - def generate_script - super + [ - init_command, - repository_command, - repository_update_command, - upgrade_command - ].compact.join("\n") - end - - private - - def upgrade_command - command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def reuse_values_flag - ['--reuse-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def version_flag - ['--version', version] - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v2/reset_command.rb b/lib/gitlab/kubernetes/helm/v2/reset_command.rb deleted file mode 100644 index 00626501a9a..00000000000 --- a/lib/gitlab/kubernetes/helm/v2/reset_command.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V2 - class ResetCommand < BaseCommand - include ClientCommand - - def generate_script - super + [ - init_command, - reset_helm_command - ].join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - private - - def reset_helm_command - 'helm reset --force' - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v3/base_command.rb b/lib/gitlab/kubernetes/helm/v3/base_command.rb deleted file mode 100644 index ca1bf5462f0..00000000000 --- a/lib/gitlab/kubernetes/helm/v3/base_command.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V3 - class BaseCommand - attr_reader :name, :files - - HELM_VERSION = '3.2.4' - - def initialize(rbac:, name:, files:) - @rbac = rbac - @name = name - @files = files - end - - def env - {} - end - - def rbac? - @rbac - end - - def pod_resource - pod_service_account_name = rbac? ? service_account_name : nil - - Gitlab::Kubernetes::Helm::Pod.new(self, namespace, service_account_name: pod_service_account_name).generate - end - - def generate_script - <<~HEREDOC - set -xeo pipefail - HEREDOC - end - - def pod_name - "install-#{name}" - end - - def config_map_resource - Gitlab::Kubernetes::ConfigMap.new(name, files).generate - end - - def service_account_resource - return unless rbac? - - Gitlab::Kubernetes::ServiceAccount.new(service_account_name, namespace).generate - end - - def cluster_role_binding_resource - return unless rbac? - - subjects = [{ kind: 'ServiceAccount', name: service_account_name, namespace: namespace }] - - Gitlab::Kubernetes::ClusterRoleBinding.new( - cluster_role_binding_name, - cluster_role_name, - subjects - ).generate - end - - def file_names - files.keys - end - - def repository_command - ['helm', 'repo', 'add', name, repository].shelljoin if repository - end - - private - - def repository_update_command - 'helm repo update' - end - - def namespace_flag - ['--namespace', Gitlab::Kubernetes::Helm::NAMESPACE] - end - - def namespace - Gitlab::Kubernetes::Helm::NAMESPACE - end - - def service_account_name - Gitlab::Kubernetes::Helm::SERVICE_ACCOUNT - end - - def cluster_role_binding_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE_BINDING - end - - def cluster_role_name - Gitlab::Kubernetes::Helm::CLUSTER_ROLE - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v3/delete_command.rb b/lib/gitlab/kubernetes/helm/v3/delete_command.rb deleted file mode 100644 index f628e852f54..00000000000 --- a/lib/gitlab/kubernetes/helm/v3/delete_command.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V3 - class DeleteCommand < BaseCommand - attr_reader :predelete, :postdelete - - def initialize(predelete: nil, postdelete: nil, **args) - super(**args) - @predelete = predelete - @postdelete = postdelete - end - - def generate_script - super + [ - predelete, - delete_command, - postdelete - ].compact.join("\n") - end - - def pod_name - "uninstall-#{name}" - end - - def delete_command - ['helm', 'uninstall', name, *namespace_flag].shelljoin - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v3/install_command.rb b/lib/gitlab/kubernetes/helm/v3/install_command.rb deleted file mode 100644 index 8d521f0dcd4..00000000000 --- a/lib/gitlab/kubernetes/helm/v3/install_command.rb +++ /dev/null @@ -1,80 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Kubernetes - module Helm - module V3 - class InstallCommand < BaseCommand - attr_reader :chart, :repository, :preinstall, :postinstall - attr_accessor :version - - def initialize(chart:, version: nil, repository: nil, preinstall: nil, postinstall: nil, **args) - super(**args) - @chart = chart - @version = version - @repository = repository - @preinstall = preinstall - @postinstall = postinstall - end - - def generate_script - super + [ - repository_command, - repository_update_command, - preinstall, - install_command, - postinstall - ].compact.join("\n") - end - - private - - # Uses `helm upgrade --install` which means we can use this for both - # installation and uprade of applications - def install_command - command = ['helm', 'upgrade', name, chart] + - install_flag + - rollback_support_flag + - reset_values_flag + - optional_version_flag + - rbac_create_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def install_flag - ['--install'] - end - - def reset_values_flag - ['--reset-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def rbac_create_flag - if rbac? - %w[--set rbac.create=true,rbac.enabled=true] - else - %w[--set rbac.create=false,rbac.enabled=false] - end - end - - def optional_version_flag - return [] unless version - - ['--version', version] - end - - def rollback_support_flag - ['--atomic', '--cleanup-on-fail'] - end - end - end - end - end -end diff --git a/lib/gitlab/kubernetes/helm/v3/patch_command.rb b/lib/gitlab/kubernetes/helm/v3/patch_command.rb deleted file mode 100644 index 1278e524bd2..00000000000 --- a/lib/gitlab/kubernetes/helm/v3/patch_command.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -# PatchCommand is for updating values in installed charts without overwriting -# existing values. -module Gitlab - module Kubernetes - module Helm - module V3 - class PatchCommand < BaseCommand - attr_reader :chart, :repository - attr_accessor :version - - def initialize(chart:, version:, repository: nil, **args) - super(**args) - - # version is mandatory to prevent chart mismatches - # we do not want our values interpreted in the context of the wrong version - raise ArgumentError, 'version is required' if version.blank? - - @chart = chart - @version = version - @repository = repository - end - - def generate_script - super + [ - repository_command, - repository_update_command, - upgrade_command - ].compact.join("\n") - end - - private - - def upgrade_command - command = ['helm', 'upgrade', name, chart] + - reuse_values_flag + - version_flag + - namespace_flag + - value_flag - - command.shelljoin - end - - def reuse_values_flag - ['--reuse-values'] - end - - def value_flag - ['-f', "/data/helm/#{name}/config/values.yaml"] - end - - def version_flag - ['--version', version] - end - end - end - end - end -end diff --git a/lib/gitlab/language_detection.rb b/lib/gitlab/language_detection.rb index b259f58350b..68deafc68b2 100644 --- a/lib/gitlab/language_detection.rb +++ b/lib/gitlab/language_detection.rb @@ -45,11 +45,11 @@ module Gitlab # Returns the ids of the programming languages that do not occur in the detection # as current repository languages def deletions - @repository_languages.map do |repo_lang| + @repository_languages.filter_map do |repo_lang| next if detection.key?(repo_lang.name) repo_lang.programming_language_id - end.compact + end end private diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index dd1502bbbcd..16c3bc09c4d 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -34,6 +34,7 @@ module Gitlab } ) end + alias_method :octokit, :api def client unless config diff --git a/lib/gitlab/legacy_github_import/importer.rb b/lib/gitlab/legacy_github_import/importer.rb index 331eab7b62a..5eeea8f9548 100644 --- a/lib/gitlab/legacy_github_import/importer.rb +++ b/lib/gitlab/legacy_github_import/importer.rb @@ -22,7 +22,7 @@ module Gitlab unless credentials raise Projects::ImportService::Error, - "Unable to find project import data credentials for project ID: #{@project.id}" + "Unable to find project import data credentials for project ID: #{@project.id}" end opts = {} @@ -55,9 +55,7 @@ module Gitlab import_comments(:issues) # Gitea doesn't have an API endpoint for pull requests comments - unless project.gitea_import? - import_comments(:pull_requests) - end + import_comments(:pull_requests) unless project.gitea_import? import_wiki @@ -67,9 +65,7 @@ module Gitlab # See: # 1) https://gitlab.com/gitlab-org/gitlab/-/issues/343448#note_985979730 # 2) https://gitlab.com/gitlab-org/gitlab/-/merge_requests/89694/diffs#dfc4a8141aa296465ea3c50b095a30292fb6ebc4_180_182 - unless project.gitea_import? - import_releases - end + import_releases unless project.gitea_import? handle_errors @@ -142,8 +138,8 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def import_pull_requests - fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |pull_requests| - pull_requests.each do |raw| + fetch_resources(:pull_requests, repo, state: :all, sort: :created, direction: :asc, per_page: 100) do |prs| + prs.each do |raw| raw = raw.to_h gh_pull_request = PullRequestFormatter.new(project, raw, client) @@ -156,11 +152,13 @@ module Gitlab merge_request = gh_pull_request.create! # Gitea doesn't return PR in the Issue API endpoint, so labels must be assigned at this stage - if project.gitea_import? - apply_labels(merge_request, raw) - end + apply_labels(merge_request, raw) if project.gitea_import? rescue StandardError => e - errors << { type: :pull_request, url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), errors: e.message } + errors << { + type: :pull_request, + url: Gitlab::UrlSanitizer.sanitize(gh_pull_request.url), + errors: e.message + } ensure clean_up_restored_branches(gh_pull_request) end @@ -196,9 +194,7 @@ module Gitlab return unless raw[:labels].count > 0 - label_ids = raw[:labels] - .map { |attrs| @labels[attrs[:name]] } - .compact + label_ids = raw[:labels].filter_map { |attrs| @labels[attrs[:name]] } issuable.update_attribute(:label_ids, label_ids) end @@ -208,10 +204,14 @@ module Gitlab resource_type = "#{issuable_type}_comments".to_sym # Two notes here: - # 1. We don't have a distinctive attribute for comments (unlike issues iid), so we fetch the last inserted note, - # compare it against every comment in the current imported page until we find match, and that's where start importing - # 2. GH returns comments for _both_ issues and PRs through issues_comments API, while pull_requests_comments returns - # only comments on diffs, so select last note not based on noteable_type but on line_code + # 1. We don't have a distinctive attribute for comments (unlike issues + # iid), so we fetch the last inserted note, compare it against every + # comment in the current imported page until we find match, and that's + # where start importing + # 2. GH returns comments for _both_ issues and PRs through + # issues_comments API, while pull_requests_comments returns only + # comments on diffs, so select last note not based on noteable_type but + # on line_code line_code_is = issuable_type == :pull_requests ? 'NOT NULL' : 'NULL' last_note = project.notes.where("line_code IS #{line_code_is}").last @@ -264,7 +264,8 @@ module Gitlab comment_attrs.with_indifferent_access == last_note_attrs end - # No matching resource in the collection, which means we got halted right on the end of the last page, so all good + # No matching resource in the collection, which means we got halted + # right on the end of the last page, so all good return unless cut_off_index # Otherwise, remove the resources we've already inserted @@ -280,9 +281,7 @@ module Gitlab # GitHub error message when the wiki repo has not been created, # this means that repo has wiki enabled, but have no pages. So, # we can skip the import. - if e.message !~ /repository not exported/ - errors << { type: :wiki, errors: e.message } - end + errors << { type: :wiki, errors: e.message } if e.message.exclude?('repository not exported') end def import_releases diff --git a/lib/gitlab/legacy_github_import/user_formatter.rb b/lib/gitlab/legacy_github_import/user_formatter.rb index d45a166d2b7..8fd8354e59c 100644 --- a/lib/gitlab/legacy_github_import/user_formatter.rb +++ b/lib/gitlab/legacy_github_import/user_formatter.rb @@ -5,6 +5,8 @@ module Gitlab class UserFormatter attr_reader :client, :raw + GITEA_GHOST_EMAIL = 'ghost_user@gitea_import_dummy_email.com' + def initialize(client, raw) @client = client @raw = raw @@ -27,7 +29,14 @@ module Gitlab private def email - @email ||= client.user(raw[:login]).to_h[:email] + # Gitea marks deleted users as 'Ghost' users and removes them from + # their system. So for Gitea 'Ghost' users we need to assign a dummy + # email address to avoid querying the Gitea api for a non existing user + if raw[:login] == 'Ghost' && raw[:id] == -1 + @email = GITEA_GHOST_EMAIL + else + @email ||= client.user(raw[:login]).to_h[:email] + end end def find_by_email diff --git a/lib/gitlab/loggable.rb b/lib/gitlab/loggable.rb new file mode 100644 index 00000000000..393e3eb37b6 --- /dev/null +++ b/lib/gitlab/loggable.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Gitlab + module Loggable + ANONYMOUS = '<Anonymous>' + + def build_structured_payload(**params) + { class: self.class.name || ANONYMOUS }.merge(params).stringify_keys + end + end +end diff --git a/lib/gitlab/mail_room.rb b/lib/gitlab/mail_room.rb index bad2e265f73..5f760e764c8 100644 --- a/lib/gitlab/mail_room.rb +++ b/lib/gitlab/mail_room.rb @@ -11,6 +11,20 @@ require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues) # This service is run independently of the main Rails process, # therefore the `Rails` class and its methods are unavailable. +# TODO: Remove this once we're on Ruby 3 +# https://gitlab.com/gitlab-org/gitlab/-/issues/393651 +unless YAML.respond_to?(:safe_load_file) + module YAML + # Temporary Ruby 2 back-compat workaround. + # + # This method only exists as of stdlib 3.0.0: + # https://ruby-doc.org/stdlib-3.0.0/libdoc/psych/rdoc/Psych.html + def self.safe_load_file(path, **options) + YAML.safe_load(File.read(path), **options) + end + end +end + module Gitlab module MailRoom RAILS_ROOT_DIR = Pathname.new('../..').expand_path(__dir__).freeze @@ -129,7 +143,7 @@ module Gitlab end def load_yaml - @yaml ||= YAML.load_file(config_file)[rails_env].deep_symbolize_keys + @yaml ||= YAML.safe_load_file(config_file, aliases: true)[rails_env].deep_symbolize_keys end def application_secrets_file diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index a4964ae0ebc..12f7c347b2d 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -64,7 +64,7 @@ module Gitlab # display_name: String, # default: Boolean }] def find_all_paths(project) - dashboards = user_facing_dashboard_services(project).flat_map do |service| + dashboards = user_facing_dashboard_services.flat_map do |service| service.all_dashboard_paths(project) end @@ -73,19 +73,8 @@ module Gitlab private - def user_facing_dashboard_services(project) - predefined_dashboard_services_for(project) + [project_service] - end - - def predefined_dashboard_services_for(project) - # Only list the self-monitoring dashboard on the self-monitoring project, - # since it is the only dashboard (at time of writing) that shows data - # about GitLab itself. - if project.self_monitoring? - return [self_monitoring_service] - end - - PREDEFINED_DASHBOARD_LIST + def user_facing_dashboard_services + PREDEFINED_DASHBOARD_LIST + [project_service] end def system_service @@ -96,10 +85,6 @@ module Gitlab ::Metrics::Dashboard::CustomDashboardService end - def self_monitoring_service - ::Metrics::Dashboard::SelfMonitoringDashboardService - end - def service_for(options) Gitlab::Metrics::Dashboard::ServiceSelector.call(options) end diff --git a/lib/gitlab/metrics/dashboard/service_selector.rb b/lib/gitlab/metrics/dashboard/service_selector.rb index 6d4b49676e5..67bf4ce7e9a 100644 --- a/lib/gitlab/metrics/dashboard/service_selector.rb +++ b/lib/gitlab/metrics/dashboard/service_selector.rb @@ -23,7 +23,6 @@ module Gitlab ::Metrics::Dashboard::DefaultEmbedService, ::Metrics::Dashboard::SystemDashboardService, ::Metrics::Dashboard::PodDashboardService, - ::Metrics::Dashboard::SelfMonitoringDashboardService, ::Metrics::Dashboard::CustomDashboardService ].freeze diff --git a/lib/gitlab/metrics/requests_rack_middleware.rb b/lib/gitlab/metrics/requests_rack_middleware.rb index f635deabf76..b4baeba72e8 100644 --- a/lib/gitlab/metrics/requests_rack_middleware.rb +++ b/lib/gitlab/metrics/requests_rack_middleware.rb @@ -22,7 +22,7 @@ module Gitlab # reasonable default. If we initialize every category we'll end up # with an explosion in unused metric combinations, but we want the # most common ones to be always present. - FEATURE_CATEGORIES_TO_INITIALIZE = ['authentication_and_authorization', + FEATURE_CATEGORIES_TO_INITIALIZE = ['system_access', 'code_review_workflow', 'continuous_integration', 'not_owned', 'source_code_management', FEATURE_CATEGORY_DEFAULT].freeze diff --git a/lib/gitlab/metrics/sidekiq_slis.rb b/lib/gitlab/metrics/sidekiq_slis.rb new file mode 100644 index 00000000000..f28cf4ac967 --- /dev/null +++ b/lib/gitlab/metrics/sidekiq_slis.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module SidekiqSlis + EXECUTION_URGENCY_DURATIONS = { + "high" => 10, + "low" => 300, + "throttled" => 300 + }.freeze + # workers without urgency attribute have "low" urgency by default in + # WorkerAttributes.get_urgency, just mirroring it here + DEFAULT_EXECUTION_URGENCY_DURATION = EXECUTION_URGENCY_DURATIONS["low"] + + class << self + def initialize_slis!(possible_labels) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_execution, possible_labels) + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:sidekiq_execution, possible_labels) + end + + def record_execution_apdex(labels, job_completion_duration) + urgency_requirement = execution_duration_for_urgency(labels[:urgency]) + Gitlab::Metrics::Sli::Apdex[:sidekiq_execution].increment( + labels: labels, + success: job_completion_duration < urgency_requirement + ) + end + + def record_execution_error(labels, error) + Gitlab::Metrics::Sli::ErrorRate[:sidekiq_execution].increment(labels: labels, error: error) + end + + def execution_duration_for_urgency(urgency) + EXECUTION_URGENCY_DURATIONS.fetch(urgency, DEFAULT_EXECUTION_URGENCY_DURATION) + end + end + end + end +end diff --git a/lib/gitlab/metrics/subscribers/action_cable.rb b/lib/gitlab/metrics/subscribers/action_cable.rb index 9f955dfe79f..3cd4b6de62b 100644 --- a/lib/gitlab/metrics/subscribers/action_cable.rb +++ b/lib/gitlab/metrics/subscribers/action_cable.rb @@ -6,13 +6,18 @@ module Gitlab class ActionCable < ActiveSupport::Subscriber include Gitlab::Utils::StrongMemoize + SOURCE_DIRECT = 'channel' + SOURCE_GRAPHQL_EVENT = 'graphql-event' + SOURCE_GRAPHQL_SUBSCRIPTION = 'graphql-subscription' + SOURCE_OTHER = 'unknown' + attach_to :action_cable SINGLE_CLIENT_TRANSMISSION = :action_cable_single_client_transmissions_total TRANSMIT_SUBSCRIPTION_CONFIRMATION = :action_cable_subscription_confirmations_total TRANSMIT_SUBSCRIPTION_REJECTION = :action_cable_subscription_rejections_total BROADCAST = :action_cable_broadcasts_total - DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes + DATA_TRANSMITTED_BYTES = :action_cable_transmitted_bytes_total def transmit_subscription_confirmation(event) confirm_subscription_counter.increment @@ -23,28 +28,57 @@ module Gitlab end def transmit(event) - transmit_counter.increment + payload = event.payload - if event.payload.present? - channel = event.payload[:channel_class] - operation = operation_name_from(event.payload) - data_size = Gitlab::Json.generate(event.payload[:data]).bytesize + labels = { + channel: payload[:channel_class], + caller: normalize_source(payload[:via]) + } + labels[:broadcasting] = graphql_event_broadcasting_from(payload[:data]) - transmitted_bytes_histogram.observe({ channel: channel, operation: operation }, data_size) - end + transmit_counter.increment(labels) + data_size = Gitlab::Json.generate(payload[:data]).bytesize + transmitted_bytes_counter.increment(labels, data_size) end def broadcast(event) - broadcast_counter.increment + broadcast_counter.increment({ broadcasting: normalize_source(event.payload[:broadcasting]) }) end private - # When possible tries to query operation name - def operation_name_from(payload) - data = payload.dig(:data, 'result', 'data') || {} + # Since transmission sources can have high dimensionality when they carry IDs, we need to + # collapse them. If it's not a well-know broadcast, we report it as "other". + def normalize_source(source) + return SOURCE_DIRECT if source.blank? + + normalized_source = source.gsub('streamed from ', '') + + if normalized_source.start_with?(SOURCE_GRAPHQL_EVENT) + # Take at most two levels of topic namespacing. + normalized_source.split(':').reject(&:empty?).take(2).join(':') # rubocop: disable CodeReuse/ActiveRecord + elsif normalized_source.start_with?(SOURCE_GRAPHQL_SUBSCRIPTION) + SOURCE_GRAPHQL_SUBSCRIPTION + else + SOURCE_OTHER + end + end - data.each_key.first + # When possible tries to query operation name. This will only return data + # for GraphQL subscription broadcasts. + def graphql_event_broadcasting_from(payload_data) + # Depending on whether the query result was passed in-process from a direct + # execution (e.g. in response to a subcription request) or cross-process by + # going through PubSub, we might encounter either string or symbol keys. + # We do not use deep_transform_keys here because the payload can be large + # and performance would be affected. + query_result = payload_data[:result] || payload_data['result'] || {} + query_result_data = query_result['data'] || {} + gql_operation = query_result_data.each_key.first + + return unless gql_operation + + "#{SOURCE_GRAPHQL_EVENT}:#{gql_operation}" end def transmit_counter @@ -83,9 +117,12 @@ module Gitlab end end - def transmitted_bytes_histogram - strong_memoize("transmitted_bytes_histogram") do - ::Gitlab::Metrics.histogram(DATA_TRANSMITTED_BYTES, 'Message size, in bytes, transmitted over action cable') + def transmitted_bytes_counter + strong_memoize("transmitted_bytes_counter") do + ::Gitlab::Metrics.counter( + DATA_TRANSMITTED_BYTES, + 'Total number of bytes transmitted over ActionCable' + ) end end end diff --git a/lib/gitlab/metrics/subscribers/active_record.rb b/lib/gitlab/metrics/subscribers/active_record.rb index e3756a8c9f6..10bb358a292 100644 --- a/lib/gitlab/metrics/subscribers/active_record.rb +++ b/lib/gitlab/metrics/subscribers/active_record.rb @@ -9,7 +9,6 @@ module Gitlab attach_to :active_record - IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze DB_COUNTERS = %i{count write_count cached_count}.freeze SQL_COMMANDS_WITH_COMMENTS_REGEX = %r{\A(/\*.*\*/\s)?((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$}i.freeze @@ -114,7 +113,7 @@ module Gitlab end def ignored_query?(payload) - payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql]) + payload[:name] == 'SCHEMA' || payload[:name] == 'TRANSACTION' end def cached_query?(payload) diff --git a/lib/gitlab/metrics/subscribers/external_http.rb b/lib/gitlab/metrics/subscribers/external_http.rb index ff8654a2cec..87756b14887 100644 --- a/lib/gitlab/metrics/subscribers/external_http.rb +++ b/lib/gitlab/metrics/subscribers/external_http.rb @@ -13,6 +13,10 @@ module Gitlab DETAIL_STORE = :external_http_detail_store COUNTER = :external_http_count DURATION = :external_http_duration_s + SLOW_REQUESTS = :external_http_slow_requests + + THRESHOLD_SLOW_REQUEST_S = 5.0 + MAX_SLOW_REQUESTS = 10 def self.detail_store ::Gitlab::SafeRequestStore[DETAIL_STORE] ||= [] @@ -26,11 +30,24 @@ module Gitlab Gitlab::SafeRequestStore[COUNTER].to_i end + def self.slow_requests + Gitlab::SafeRequestStore[SLOW_REQUESTS] + end + + def self.top_slowest_requests + requests = slow_requests + + return unless requests.present? + + requests.sort_by { |req| req[:duration_s] }.reverse.first(MAX_SLOW_REQUESTS) + end + def self.payload { COUNTER => request_count, - DURATION => duration - } + DURATION => duration, + SLOW_REQUESTS => top_slowest_requests + }.compact end def request(event) @@ -69,6 +86,17 @@ module Gitlab Gitlab::SafeRequestStore[COUNTER] = Gitlab::SafeRequestStore[COUNTER].to_i + 1 Gitlab::SafeRequestStore[DURATION] = Gitlab::SafeRequestStore[DURATION].to_f + payload[:duration].to_f + + if payload[:duration].to_f > THRESHOLD_SLOW_REQUEST_S + Gitlab::SafeRequestStore[SLOW_REQUESTS] ||= [] + Gitlab::SafeRequestStore[SLOW_REQUESTS] << { + method: payload[:method], + host: payload[:host], + port: payload[:port], + path: payload[:path], + duration_s: payload[:duration].to_f.round(3) + } + end end def expose_metrics(payload) diff --git a/lib/gitlab/metrics/subscribers/rack_attack.rb b/lib/gitlab/metrics/subscribers/rack_attack.rb index e6cf14a6c8c..2196122df01 100644 --- a/lib/gitlab/metrics/subscribers/rack_attack.rb +++ b/lib/gitlab/metrics/subscribers/rack_attack.rb @@ -22,11 +22,6 @@ module Gitlab } end - def redis(event) - self.class.payload[:rack_attack_redis_count] += 1 - self.class.payload[:rack_attack_redis_duration_s] += event.duration.to_f / 1000 - end - def safelist(event) req = event.payload[:request] Gitlab::Instrumentation::Throttle.safelist = req.env['rack.attack.matched'] diff --git a/lib/gitlab/metrics/subscribers/rails_cache.rb b/lib/gitlab/metrics/subscribers/rails_cache.rb index b12db9df66d..b4e9e85a012 100644 --- a/lib/gitlab/metrics/subscribers/rails_cache.rb +++ b/lib/gitlab/metrics/subscribers/rails_cache.rb @@ -9,18 +9,19 @@ module Gitlab attach_to :active_support def cache_read_multi(event) - observe(:read_multi, event.duration) + observe(:read_multi, event) return unless current_transaction - current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size) do + labels = { store: event.payload[:store].split('::').last } + current_transaction.observe(:gitlab_cache_read_multikey_count, event.payload[:key].size, labels) do buckets [10, 50, 100, 1000] docstring 'Number of keys for mget in read_multi/fetch_multi' end end def cache_read(event) - observe(:read, event.duration) + observe(:read, event) return unless current_transaction return if event.payload[:super_operation] == :fetch @@ -33,15 +34,15 @@ module Gitlab end def cache_write(event) - observe(:write, event.duration) + observe(:write, event) end def cache_delete(event) - observe(:delete, event.duration) + observe(:delete, event) end def cache_exist?(event) - observe(:exists, event.duration) + observe(:exists, event) end def cache_fetch_hit(event) @@ -60,17 +61,17 @@ module Gitlab current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1) end - def observe(key, duration) + def observe(key, event) return unless current_transaction - labels = { operation: key } + labels = { operation: key, store: event.payload[:store].split('::').last } current_transaction.increment(:gitlab_cache_operations_total, 1, labels) do docstring 'Cache operations' label_keys labels.keys end - metric_cache_operation_duration_seconds.observe(labels, duration / 1000.0) + metric_cache_operation_duration_seconds.observe(labels, event.duration / 1000.0) end private @@ -84,7 +85,7 @@ module Gitlab :gitlab_cache_operation_duration_seconds, 'Cache access time', {}, - [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0] + Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS ) end end diff --git a/lib/gitlab/middleware/compressed_json.rb b/lib/gitlab/middleware/compressed_json.rb index 80916eab5ac..cc485d8a5db 100644 --- a/lib/gitlab/middleware/compressed_json.rb +++ b/lib/gitlab/middleware/compressed_json.rb @@ -4,15 +4,23 @@ module Gitlab module Middleware class CompressedJson COLLECTOR_PATH = '/api/v4/error_tracking/collector' - PACKAGES_PATH = %r{ - \A/api/v4/ (?# prefix) - (?:projects/ - (?<project_id> - .+ (?# at least one character) - )/ - )? (?# projects segment) - packages/npm/-/npm/v1/security/ - (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + INSTANCE_PACKAGES_PATH = %r{ + \A/api/v4/packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + }xi.freeze + GROUP_PACKAGES_PATH = %r{ + \A/api/v4/groups/ + (?<id> + [a-zA-Z0-9%-._]{1,255} + )/-/packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) + }xi.freeze + PROJECT_PACKAGES_PATH = %r{ + \A/api/v4/projects/ + (?<id> + [a-zA-Z0-9%-._]{1,255} + )/packages/npm/-/npm/v1/security/ + (?:(?:advisories/bulk)|(?:audits/quick))\z (?# end) }xi.freeze MAXIMUM_BODY_SIZE = 200.kilobytes.to_i UNSAFE_CHARACTERS = %r{[!"#&'()*+,./:;<>=?@\[\]^`{}|~$]}xi.freeze @@ -76,16 +84,19 @@ module Gitlab end def match_packages_path?(env) - match_data = env['PATH_INFO'].delete_prefix(relative_url).match(PACKAGES_PATH) + path = env['PATH_INFO'].delete_prefix(relative_url) + match_data = path.match(INSTANCE_PACKAGES_PATH) || + path.match(PROJECT_PACKAGES_PATH) || + path.match(GROUP_PACKAGES_PATH) return false unless match_data - return true unless match_data[:project_id] # instance level endpoint was matched + return true if match_data.names.empty? # instance level endpoint was matched - url_encoded?(match_data[:project_id]) + url_encoded?(match_data[:id]) end - def url_encoded?(project_id) - project_id !~ UNSAFE_CHARACTERS + def url_encoded?(id) + id !~ UNSAFE_CHARACTERS end end end diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index 13f7ab36823..4da5fef9fd7 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -18,7 +18,7 @@ module Gitlab request = ActionDispatch::Request.new(env) render_go_doc(request) || @app.call(env) - rescue Gitlab::Auth::IpBlacklisted + rescue Gitlab::Auth::IpBlocked Gitlab::AuthLogger.error( message: 'Rack_Attack', status: 403, diff --git a/lib/gitlab/middleware/request_context.rb b/lib/gitlab/middleware/request_context.rb index 07f6f87a68c..f609002007c 100644 --- a/lib/gitlab/middleware/request_context.rb +++ b/lib/gitlab/middleware/request_context.rb @@ -8,15 +8,9 @@ module Gitlab end def call(env) - # We should be using ActionDispatch::Request instead of - # Rack::Request to be consistent with Rails, but due to a Rails - # bug described in - # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 - # hosts behind a load balancer will only see 127.0.0.1 for the - # load balancer's IP. - req = Rack::Request.new(env) - - ::Gitlab::InstrumentationHelper.init_instrumentation_data(request_ip: req.ip) + request = ActionDispatch::Request.new(env) + Gitlab::RequestContext.start_request_context(request: request) + Gitlab::RequestContext.start_thread_context @app.call(env) end diff --git a/lib/gitlab/multi_collection_paginator.rb b/lib/gitlab/multi_collection_paginator.rb index 87cc0a0d3d2..e122f0b9317 100644 --- a/lib/gitlab/multi_collection_paginator.rb +++ b/lib/gitlab/multi_collection_paginator.rb @@ -9,6 +9,8 @@ module Gitlab @per_page = (per_page || Kaminari.config.default_per_page).to_i @first_collection, @second_collection = collections + + raise ArgumentError, 'Page size must be at least 1' if @per_page < 1 end def paginate(page) diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index 75eb0e8a264..a83cdbe15df 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -8,7 +8,10 @@ module Gitlab # this is already :/. We could also take a hash and manually check every # entry, but it's much more maintainable to do rely on native Ruby. # rubocop: disable Metrics/ParameterLists - def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) + def self.build( + id:, title:, active: false, icon: '', href: '', view: '', + css_class: nil, data: nil, partial: nil, component: nil + ) { id: id, type: :item, @@ -19,7 +22,8 @@ module Gitlab view: view.to_s, css_class: css_class, data: data || { qa_selector: 'menu_item_link', qa_title: title }, - emoji: emoji + partial: partial, + component: component } end # rubocop: enable Metrics/ParameterLists diff --git a/lib/gitlab/no_cache_headers.rb b/lib/gitlab/no_cache_headers.rb index 2d03741480d..6afb108dcfd 100644 --- a/lib/gitlab/no_cache_headers.rb +++ b/lib/gitlab/no_cache_headers.rb @@ -4,7 +4,6 @@ module Gitlab module NoCacheHeaders DEFAULT_GITLAB_NO_CACHE_HEADERS = { 'Cache-Control' => "#{ActionDispatch::Http::Cache::Response::DEFAULT_CACHE_CONTROL}, no-store, no-cache", - 'Pragma' => 'no-cache', # HTTP 1.0 compatibility 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' }.freeze diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index 8dbd2f41ccb..f7f65c91339 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -2,8 +2,19 @@ module Gitlab module Observability - module_function + extend self + ACTION_TO_PERMISSION = { + explore: :read_observability, + datasources: :admin_observability, + manage: :admin_observability, + dashboards: :read_observability + }.freeze + + EMBEDDABLE_PATHS = %w[explore goto].freeze + + # Returns the GitLab Observability URL + # def observability_url return ENV['OVERRIDE_OBSERVABILITY_URL'] if ENV['OVERRIDE_OBSERVABILITY_URL'] # TODO Make observability URL configurable https://gitlab.com/gitlab-org/opstrace/opstrace-ui/-/issues/80 @@ -12,8 +23,123 @@ module Gitlab 'https://observe.gitlab.com' end - def observability_enabled?(user, group) - Gitlab::Observability.observability_url.present? && Ability.allowed?(user, :read_observability, group) + # Returns true if the Observability feature flag is enabled + # + def enabled?(group = nil) + return Feature.enabled?(:observability_group_tab, group) if group + + Feature.enabled?(:observability_group_tab) + end + + # Returns the embeddable Observability URL of a given URL + # + # - Validates the URL + # - Checks that the path is embeddable + # - Converts the gitlab.com URL to observe.gitlab.com URL + # + # e.g. + # + # from: gitlab.com/groups/GROUP_PATH/-/observability/explore?observability_path=/explore + # to observe.gitlab.com/-/GROUP_ID/explore + # + # Returns nil if the URL is not a valid Observability URL or the path is not embeddable + # + def embeddable_url(url) + uri = validate_url(url, Gitlab.config.gitlab.url) + return unless uri + + group = group_from_observability_url(url) + return unless group + + parsed_query = CGI.parse(uri.query.to_s).transform_values(&:first).symbolize_keys + observability_path = parsed_query[:observability_path] + + return build_full_url(group, observability_path, '/') if observability_path_embeddable?(observability_path) + end + + # Returns true if the user is allowed to perform an action within a group + # + def allowed_for_action?(user, group, action) + return false if action.nil? + + permission = ACTION_TO_PERMISSION.fetch(action.to_sym, :admin_observability) + allowed?(user, group, permission) + end + + # Returns true if the user has the specified permission within the group + def allowed?(user, group, permission = :admin_observability) + return false unless group && user + + observability_url.present? && Ability.allowed?(user, permission, group) + end + + # Builds the full Observability URL given a certan group and path + # + # If unsanitized_observability_path is not valid or missing, fallbacks to fallback_path + # + def build_full_url(group, unsanitized_observability_path, fallback_path) + return unless group + + # When running Observability UI in standalone mode (i.e. not backed by Observability Backend) + # the group-id is not required. !!This is only used for local dev!! + base_url = ENV['STANDALONE_OBSERVABILITY_UI'] == 'true' ? observability_url : "#{observability_url}/-/#{group.id}" + + sanitized_path = if unsanitized_observability_path && sanitize(unsanitized_observability_path) != '' + CGI.unescapeHTML(sanitize(unsanitized_observability_path)) + else + fallback_path || '/' + end + + sanitized_path.prepend('/') if sanitized_path[0] != '/' + + "#{base_url}#{sanitized_path}" + end + + private + + def validate_url(url, reference_url) + uri = URI.parse(url) + reference_uri = URI.parse(reference_url) + + return uri if uri.scheme == reference_uri.scheme && + uri.port == reference_uri.port && + uri.host.casecmp?(reference_uri.host) + rescue URI::InvalidURIError + nil + end + + def link_sanitizer + @link_sanitizer ||= Rails::Html::Sanitizer.safe_list_sanitizer.new + end + + def sanitize(input) + link_sanitizer.sanitize(input, {})&.html_safe + end + + def group_from_observability_url(url) + match = Rails.application.routes.recognize_path(url) + + return if match[:unmatched_route].present? + return if match[:group_id].blank? || match[:action].blank? || match[:controller] != "groups/observability" + + group_path = match[:group_id] + Group.find_by_full_path(group_path) + rescue ActionController::RoutingError + nil + end + + def observability_path_embeddable?(observability_path) + return false unless observability_path + + observability_path = observability_path[1..] if observability_path[0] == '/' + + parsed_observability_path = URI.parse(observability_path).path.split('/') + + base_path = parsed_observability_path[0] + + EMBEDDABLE_PATHS.include?(base_path) + rescue URI::InvalidURIError + false end end end diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index a92860f7eb8..f944f9827a3 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -11,7 +11,8 @@ module Gitlab Gitlab::UrlBlocker.validate!(env[:url], schemes: %w[http https], allow_localhost: allow_local_requests?, - allow_local_network: allow_local_requests? + allow_local_network: allow_local_requests?, + dns_rebind_protection: dns_rebind_protection? ) @app.call(env) @@ -19,6 +20,10 @@ module Gitlab private + def dns_rebind_protection? + Gitlab::CurrentSettings.dns_rebinding_protection_enabled? + end + def allow_local_requests? Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? end diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index a03533dcd9a..81ad7a7f9e1 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -21,8 +21,6 @@ module Gitlab class << self def default_arguments_for(provider_name) case provider_name - when 'cas3' - { on_single_sign_out: cas3_signout_handler } when 'shibboleth' { fail_with_empty_uid: true } when 'google_oauth2' @@ -39,18 +37,6 @@ module Gitlab def full_host proc { |_env| Settings.gitlab['base_url'] } end - - private - - def cas3_signout_handler - lambda do |request| - ticket = request.params[:session_index] - raise "Service Ticket not found." unless Gitlab::Auth::OAuth::Session.valid?(:cas3, ticket) - - Gitlab::Auth::OAuth::Session.destroy(:cas3, ticket) - true - end - end end private @@ -74,7 +60,7 @@ module Gitlab # An Array from the configuration will be expanded provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? - when Hash + when Hash, GitlabSettings::Options hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) normalized = normalize_hash_arguments(hash_arguments) diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 9f39b5f122f..3c8ac55f70b 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -10,8 +10,11 @@ module Gitlab start_time = Gitlab::Metrics::System.monotonic_time retry_attempts = 0 + # prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186 + klass = subject.is_a?(ActiveRecord::Relation) ? subject.klass : subject.class + begin - subject.transaction do + klass.transaction do yield(subject) end rescue ActiveRecord::StaleObjectError diff --git a/lib/gitlab/other_markup.rb b/lib/gitlab/other_markup.rb index 2368ea3ad28..5b6be88a46f 100644 --- a/lib/gitlab/other_markup.rb +++ b/lib/gitlab/other_markup.rb @@ -22,15 +22,10 @@ module Gitlab Gitlab::RenderTimeout.timeout(foreground: RENDER_TIMEOUT) { GitHub::Markup.render(file_name, input) } rescue Timeout::Error => e class_name = name.demodulize - timeout_counter.increment(source: class_name) Gitlab::ErrorTracking.track_exception(e, project_id: context[:project]&.id, class_name: class_name, file_name: file_name) ActionController::Base.helpers.simple_format(input) end - - def self.timeout_counter - Gitlab::Metrics.counter(:banzai_filter_timeouts_total, 'Count of the Banzai filters that time out') - end end end diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb index 2f5c6938e2a..6845f5d88ec 100644 --- a/lib/gitlab/pages/deployment_update.rb +++ b/lib/gitlab/pages/deployment_update.rb @@ -46,9 +46,13 @@ module Gitlab end end + def root_dir + build.options[:publish] || PUBLIC_DIR + end + # Calculate page size after extract def total_size - @total_size ||= build.artifacts_metadata_entry(PUBLIC_DIR + '/', recursive: true).total_size + @total_size ||= build.artifacts_metadata_entry("#{root_dir}/", recursive: true).total_size end def max_size_from_settings @@ -74,7 +78,10 @@ module Gitlab def validate_public_folder if total_size <= 0 - errors.add(:base, 'Error: The `public/` folder is missing, or not declared in `.gitlab-ci.yml`.') + errors.add( + :base, + 'Error: You need to either include a `public/` folder in your artifacts, or specify ' \ + 'which one to use for Pages using `publish` in `.gitlab-ci.yml`') end end diff --git a/lib/gitlab/pages/random_domain.rb b/lib/gitlab/pages/random_domain.rb new file mode 100644 index 00000000000..8aa7611c910 --- /dev/null +++ b/lib/gitlab/pages/random_domain.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class RandomDomain + PROJECT_PATH_LIMIT = 48 + SUBDOMAIN_LABEL_LIMIT = 63 + + def self.generate(project_path:, namespace_path:) + new(project_path: project_path, namespace_path: namespace_path).generate + end + + def initialize(project_path:, namespace_path:) + @project_path = project_path + @namespace_path = namespace_path + end + + # Subdomains have a limit of 63 bytes (https://www.freesoft.org/CIE/RFC/1035/9.htm) + # For this reason we're limiting each part of the unique subdomain + # + # The domain is made up of 3 parts, like: projectpath-namespacepath-randomstring + # - project path: between 1 and 48 chars + # - namespace path: when the project path has less than 48 chars, + # the namespace full path will be used to fill the value up to 48 chars + # - random hexadecimal: to ensure a random value, the domain is then filled + # with a random hexadecimal value to complete 63 chars + def generate + domain = project_path.byteslice(0, PROJECT_PATH_LIMIT) + + # if the project_path has less than PROJECT_PATH_LIMIT chars, + # fill the domain with the parent full_path up to 48 chars like: + # projectpath-namespacepath + if domain.length < PROJECT_PATH_LIMIT + namespace_size = PROJECT_PATH_LIMIT - domain.length - 1 + domain.concat('-', namespace_path.byteslice(0, namespace_size)) + end + + # Complete the domain with random hexadecimal values util it is 63 chars long + # PS.: SecureRandom.hex return an string twice the size passed as argument. + domain.concat('-', SecureRandom.hex(SUBDOMAIN_LABEL_LIMIT - domain.length - 1)) + + # Slugify ensures the format and size (63 chars) of the given string + Gitlab::Utils.slugify(domain) + end + + private + + attr_reader :project_path, :namespace_path + end + end +end diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb new file mode 100644 index 00000000000..5fec60188f8 --- /dev/null +++ b/lib/gitlab/pages/virtual_host_finder.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class VirtualHostFinder + def initialize(host) + @host = host&.downcase + end + + def execute + return if host.blank? + + gitlab_host = ::Settings.pages.host.downcase.prepend(".") + + if host.ends_with?(gitlab_host) + name = host.delete_suffix(gitlab_host) + + by_namespace_domain(name) || + by_unique_domain(name) + else + by_custom_domain(host) + end + end + + private + + attr_reader :host + + def by_unique_domain(name) + project = Project.by_pages_enabled_unique_domain(name) + + return unless Feature.enabled?(:pages_unique_domain, project) + return unless project&.pages_deployed? + + ::Pages::VirtualDomain.new(projects: [project]) + end + + def by_namespace_domain(name) + namespace = Namespace.top_most.by_path(name) + + return if namespace.blank? + + cache = if Feature.enabled?(:cache_pages_domain_api, namespace) + ::Gitlab::Pages::CacheControl.for_namespace(namespace.id) + end + + ::Pages::VirtualDomain.new( + trim_prefix: namespace.full_path, + projects: namespace.all_projects_with_pages, + cache: cache + ) + end + + def by_custom_domain(host) + domain = PagesDomain.find_by_domain_case_insensitive(host) + + return unless domain&.pages_deployed? + + cache = if Feature.enabled?(:cache_pages_domain_api, domain.project.root_namespace) + ::Gitlab::Pages::CacheControl.for_domain(domain.id) + end + + ::Pages::VirtualDomain.new( + projects: [domain.project], + domain: domain, + cache: cache + ) + end + end + end +end diff --git a/lib/gitlab/pagination/keyset.rb b/lib/gitlab/pagination/keyset.rb index 67a5530d46c..56017ba846c 100644 --- a/lib/gitlab/pagination/keyset.rb +++ b/lib/gitlab/pagination/keyset.rb @@ -3,12 +3,12 @@ module Gitlab module Pagination module Keyset - SUPPORTED_TYPES = [ + SUPPORTED_TYPES = %w[ Project ].freeze def self.available_for_type?(relation) - SUPPORTED_TYPES.include?(relation.klass) + SUPPORTED_TYPES.include?(relation.klass.to_s) end def self.available?(request_context, relation) diff --git a/lib/gitlab/pagination/keyset/paginator.rb b/lib/gitlab/pagination/keyset/paginator.rb index 1ff4589d8e1..fb670a3f9ea 100644 --- a/lib/gitlab/pagination/keyset/paginator.rb +++ b/lib/gitlab/pagination/keyset/paginator.rb @@ -94,8 +94,6 @@ module Gitlab data = order.cursor_attributes_for_node(records.last) data[direction_key] = FORWARD_DIRECTION cursor_converter.dump(data) - else - nil end end diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index 20d8f7be8fd..8a7566f6e0e 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # The purpose of this code is to set the migrations path -# for the Geo tracking database. +# for the Geo tracking database and the embedding database. module Gitlab module Patch module DatabaseConfig @@ -10,13 +10,17 @@ module Gitlab def database_configuration super.to_h do |env, configs| if Gitlab.ee? - if configs.key?("geo") - migrations_paths = Array(configs["geo"]["migrations_paths"]) - migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? - migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + ee_databases = %w[embedding geo] - configs["geo"]["migrations_paths"] = migrations_paths.uniq - configs["geo"]["schema_migrations_path"] = "ee/db/geo/schema_migrations" if configs["geo"]["schema_migrations_path"].blank? + ee_databases.each do |ee_db_name| + next unless configs.key?(ee_db_name) + + migrations_paths = Array(configs[ee_db_name]['migrations_paths']) + migrations_paths << File.join('ee', 'db', ee_db_name, 'migrate') if migrations_paths.empty? + migrations_paths << File.join('ee', 'db', ee_db_name, 'post_migrate') unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + + configs[ee_db_name]['migrations_paths'] = migrations_paths.uniq + configs[ee_db_name]['schema_migrations_path'] = File.join('ee', 'db', ee_db_name, 'schema_migrations') if configs[ee_db_name]['schema_migrations_path'].blank? end end diff --git a/lib/gitlab/patch/draw_route.rb b/lib/gitlab/patch/draw_route.rb index 61b25065e8f..67da6c9c943 100644 --- a/lib/gitlab/patch/draw_route.rb +++ b/lib/gitlab/patch/draw_route.rb @@ -27,7 +27,7 @@ module Gitlab def draw_route(path) if File.exist?(path) - instance_eval(File.read(path)) + instance_eval(File.read(path), path.to_s) true else false diff --git a/lib/gitlab/patch/node_loader.rb b/lib/gitlab/patch/node_loader.rb new file mode 100644 index 00000000000..79f4b17dd93 --- /dev/null +++ b/lib/gitlab/patch/node_loader.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2212#note_1287996694 +# It uses hostname instead of IP address if the former is present in `CLUSTER NODES` output. +if Gem::Version.new(Redis::VERSION) > Gem::Version.new('4.8.1') + raise 'New version of redis detected, please remove or update this patch' +end + +module Gitlab + module Patch + module NodeLoader + def self.prepended(base) + base.class_eval do + # monkey-patches https://github.com/redis/redis-rb/blob/v4.8.0/lib/redis/cluster/node_loader.rb#L23 + def self.fetch_node_info(node) + node.call(%i[cluster nodes]).split("\n").map(&:split).to_h do |arr| + [ + extract_host_identifier(arr[1]), + (arr[2].split(',') & %w[master slave]).first # rubocop:disable Naming/InclusiveLanguage + ] + end + end + + # Since `CLUSTER SLOT` uses the preferred endpoint determined by + # the `cluster-preferred-endpoint-type` config value, we will prefer hostname over IP address. + # See https://redis.io/commands/cluster-nodes/ for details on the output format. + # + # @param [String] Address info matching fhe format: <ip:port@cport[,hostname[,auxiliary_field=value]*]> + def self.extract_host_identifier(node_address) + ip_chunk, hostname, _auxiliaries = node_address.split(',') + return ip_chunk.split('@').first if hostname.blank? + + port = ip_chunk.split('@').first.split(':')[1] + "#{hostname}:#{port}" + end + end + end + end + end +end diff --git a/lib/gitlab/phabricator_import.rb b/lib/gitlab/phabricator_import.rb deleted file mode 100644 index 49e01eceb5b..00000000000 --- a/lib/gitlab/phabricator_import.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - BaseError = Class.new(StandardError) - - def self.available? - Gitlab::CurrentSettings.import_sources.include?('phabricator') - end - end -end diff --git a/lib/gitlab/phabricator_import/cache/map.rb b/lib/gitlab/phabricator_import/cache/map.rb deleted file mode 100644 index 7aba3cf26fd..00000000000 --- a/lib/gitlab/phabricator_import/cache/map.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Cache - class Map - def initialize(project) - @project = project - end - - def get_gitlab_model(phabricator_id) - cached_info = get(phabricator_id) - - if cached_info[:classname] && cached_info[:database_id] - object = cached_info[:classname].constantize.find_by_id(cached_info[:database_id]) - else - object = yield if block_given? - set_gitlab_model(object, phabricator_id) if object - end - - object - end - - def set_gitlab_model(object, phabricator_id) - set(object.class, object.id, phabricator_id) - end - - private - - attr_reader :project - - def set(klass_name, object_id, phabricator_id) - key = cache_key_for_phabricator_id(phabricator_id) - - redis.with do |r| - r.multi do |multi| - multi.mapped_hmset(key, - { classname: klass_name, database_id: object_id }) - multi.expire(key, timeout) - end - end - end - - def get(phabricator_id) - key = cache_key_for_phabricator_id(phabricator_id) - - redis.with do |r| - r.pipelined do |pipe| - # Extend the TTL when a key was - pipe.expire(key, timeout) - pipe.mapped_hmget(key, :classname, :database_id) - end.last - end - end - - def cache_key_for_phabricator_id(phabricator_id) - "#{Redis::Cache::CACHE_NAMESPACE}/phabricator-import/#{project.id}/#{phabricator_id}" - end - - def redis - Gitlab::Redis::Cache - end - - def timeout - # Setting the timeout to the same one as we do for clearing stuck jobs - # this makes sure all cache is available while the import is running. - Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit.rb b/lib/gitlab/phabricator_import/conduit.rb deleted file mode 100644 index 4c64d737389..00000000000 --- a/lib/gitlab/phabricator_import/conduit.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - ApiError = Class.new(Gitlab::PhabricatorImport::BaseError) - ResponseError = Class.new(ApiError) - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/client.rb b/lib/gitlab/phabricator_import/conduit/client.rb deleted file mode 100644 index 5945cde9618..00000000000 --- a/lib/gitlab/phabricator_import/conduit/client.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class Client - def initialize(phabricator_url, api_token) - @phabricator_url = phabricator_url - @api_token = api_token - end - - def get(path, params: {}) - response = Gitlab::HTTP.get(build_url(path), body: build_params(params), headers: headers) - Response.parse!(response) - rescue *Gitlab::HTTP::HTTP_ERRORS => e - # Wrap all errors from the API into an API-error. - raise ApiError, e - end - - private - - attr_reader :phabricator_url, :api_token - - def headers - { "Accept" => 'application/json' } - end - - def build_url(path) - URI.join(phabricator_url, '/api/', path).to_s - end - - def build_params(params) - params = params.dup - params.compact! - params.reverse_merge!("api.token" => api_token) - - CGI.unescape(params.to_query) - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/maniphest.rb b/lib/gitlab/phabricator_import/conduit/maniphest.rb deleted file mode 100644 index 848b71e49e7..00000000000 --- a/lib/gitlab/phabricator_import/conduit/maniphest.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class Maniphest - def initialize(phabricator_url:, api_token:) - @client = Client.new(phabricator_url, api_token) - end - - def tasks(after: nil) - TasksResponse.new(get_tasks(after)) - end - - private - - def get_tasks(after) - client.get('maniphest.search', - params: { - after: after, - attachments: { projects: 1, subscribers: 1, columns: 1 } - }) - end - - attr_reader :client - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/pagination.rb b/lib/gitlab/phabricator_import/conduit/pagination.rb deleted file mode 100644 index 5f54cccdbc8..00000000000 --- a/lib/gitlab/phabricator_import/conduit/pagination.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class Pagination - def initialize(cursor_json) - @cursor_json = cursor_json - end - - def has_next_page? - next_page.present? - end - - def next_page - cursor_json["after"] - end - - private - - attr_reader :cursor_json - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/response.rb b/lib/gitlab/phabricator_import/conduit/response.rb deleted file mode 100644 index 26037ba183e..00000000000 --- a/lib/gitlab/phabricator_import/conduit/response.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class Response - def self.parse!(http_response) - unless http_response.success? - raise Gitlab::PhabricatorImport::Conduit::ResponseError, - "Phabricator responded with #{http_response.status}" - end - - response = new(Gitlab::Json.parse(http_response.body)) - - unless response.success? - raise ResponseError, - "Phabricator Error: #{response.error_code}: #{response.error_info}" - end - - response - rescue JSON::JSONError => e - raise ResponseError, e - end - - def initialize(json) - @json = json - end - - def success? - error_code.nil? - end - - def error_code - json['error_code'] - end - - def error_info - json['error_info'] - end - - def data - json_result&.fetch('data') - end - - def pagination - return unless cursor_info = json_result&.fetch('cursor') - - @pagination ||= Pagination.new(cursor_info) - end - - private - - attr_reader :json - - def json_result - json['result'] - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/tasks_response.rb b/lib/gitlab/phabricator_import/conduit/tasks_response.rb deleted file mode 100644 index cbcf7259fb2..00000000000 --- a/lib/gitlab/phabricator_import/conduit/tasks_response.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class TasksResponse - def initialize(conduit_response) - @conduit_response = conduit_response - end - - delegate :pagination, to: :conduit_response - - def tasks - @tasks ||= conduit_response.data.map do |task_json| - Gitlab::PhabricatorImport::Representation::Task.new(task_json) - end - end - - private - - attr_reader :conduit_response - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/user.rb b/lib/gitlab/phabricator_import/conduit/user.rb deleted file mode 100644 index fc8c3f7cde9..00000000000 --- a/lib/gitlab/phabricator_import/conduit/user.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Conduit - class User - MAX_PAGE_SIZE = 100 - - def initialize(phabricator_url:, api_token:) - @client = Client.new(phabricator_url, api_token) - end - - def users(phids) - phids.each_slice(MAX_PAGE_SIZE).map { |limited_phids| get_page(limited_phids) } - end - - private - - def get_page(phids) - UsersResponse.new(get_users(phids)) - end - - def get_users(phids) - client.get('user.search', - params: { constraints: { phids: phids } }) - end - - attr_reader :client - end - end - end -end diff --git a/lib/gitlab/phabricator_import/conduit/users_response.rb b/lib/gitlab/phabricator_import/conduit/users_response.rb deleted file mode 100644 index 3dfb29a7be5..00000000000 --- a/lib/gitlab/phabricator_import/conduit/users_response.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - module Conduit - class UsersResponse - def initialize(conduit_response) - @conduit_response = conduit_response - end - - def users - @users ||= conduit_response.data.map do |user_json| - Gitlab::PhabricatorImport::Representation::User.new(user_json) - end - end - - private - - attr_reader :conduit_response - end - end - end -end diff --git a/lib/gitlab/phabricator_import/importer.rb b/lib/gitlab/phabricator_import/importer.rb deleted file mode 100644 index 0666fa0df01..00000000000 --- a/lib/gitlab/phabricator_import/importer.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - class Importer - def self.async? - true - end - - def self.imports_repository? - # This does not really import a repository, but we want to skip all - # repository related tasks in the `Projects::ImportService` - true - end - - def initialize(project) - @project = project - end - - def execute - Gitlab::Import::SetAsyncJid.set_jid(project.import_state) - schedule_first_tasks_page - - true - rescue StandardError => e - fail_import(e.message) - - false - end - - private - - attr_reader :project - - def schedule_first_tasks_page - ImportTasksWorker.schedule(project.id) - end - - def fail_import(message) - project.import_state.mark_as_failed(message) - end - end - end -end diff --git a/lib/gitlab/phabricator_import/issues/importer.rb b/lib/gitlab/phabricator_import/issues/importer.rb deleted file mode 100644 index 478c26af030..00000000000 --- a/lib/gitlab/phabricator_import/issues/importer.rb +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Issues - class Importer - def initialize(project, after = nil) - @project = project - @after = after - end - - def execute - schedule_next_batch - - tasks_response.tasks.each do |task| - TaskImporter.new(project, task).execute - end - end - - private - - attr_reader :project, :after - - def schedule_next_batch - return unless tasks_response.pagination.has_next_page? - - Gitlab::PhabricatorImport::ImportTasksWorker - .schedule(project.id, tasks_response.pagination.next_page) - end - - def tasks_response - @tasks_response ||= client.tasks(after: after) - end - - def client - @client ||= - Gitlab::PhabricatorImport::Conduit::Maniphest - .new(phabricator_url: project.import_data.data['phabricator_url'], - api_token: project.import_data.credentials[:api_token]) - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/issues/task_importer.rb b/lib/gitlab/phabricator_import/issues/task_importer.rb deleted file mode 100644 index 9c419ecb700..00000000000 --- a/lib/gitlab/phabricator_import/issues/task_importer.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Issues - class TaskImporter - def initialize(project, task) - @project = project - @task = task - end - - def execute - issue.author = user_finder.find(task.author_phid) || User.ghost - - # TODO: Reformat the description with attachments, escaping accidental - # links and add attachments - # https://gitlab.com/gitlab-org/gitlab-foss/issues/60603 - issue.assign_attributes(task.issue_attributes) - - save! - - if owner = user_finder.find(task.owner_phid) - issue.assignees << owner - end - - issue - end - - private - - attr_reader :project, :task - - def save! - # Just avoiding an extra redis call, we've already updated the expiry - # when reading the id from the map - was_persisted = issue.persisted? - - issue.save! if issue.changed? - - object_map.set_gitlab_model(issue, task.phabricator_id) unless was_persisted - end - - def issue - @issue ||= find_issue_by_phabricator_id(task.phabricator_id) || - project.issues.new - end - - def user_finder - @issue_finder ||= Gitlab::PhabricatorImport::UserFinder.new(project, task.phids) - end - - def find_issue_by_phabricator_id(phabricator_id) - object_map.get_gitlab_model(phabricator_id) - end - - def object_map - Gitlab::PhabricatorImport::Cache::Map.new(project) - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/project_creator.rb b/lib/gitlab/phabricator_import/project_creator.rb deleted file mode 100644 index 4de9eaa9500..00000000000 --- a/lib/gitlab/phabricator_import/project_creator.rb +++ /dev/null @@ -1,77 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - class ProjectCreator - def initialize(current_user, params) - @current_user = current_user - @params = params.dup - end - - def execute - return unless import_url.present? && api_token.present? - - project = Projects::CreateService.new(current_user, create_params).execute - return project unless project.persisted? - - project.project_feature.update!(project_feature_attributes) - - project - end - - private - - attr_reader :current_user, :params - - def create_params - { - name: project_name, - path: project_path, - namespace_id: namespace_id, - import_type: 'phabricator', - import_url: Project::UNKNOWN_IMPORT_URL, - import_data: import_data - } - end - - def project_name - params[:name] - end - - def project_path - params[:path] - end - - def namespace_id - params[:namespace_id] || current_user.namespace_id - end - - def import_url - params[:phabricator_server_url] - end - - def api_token - params[:api_token] - end - - def project_feature_attributes - # everything disabled except for issues - @project_features_attributes ||= - ProjectFeature::FEATURES.to_h do |feature| - [ProjectFeature.access_level_attribute(feature), ProjectFeature::DISABLED] - end.merge(ProjectFeature.access_level_attribute(:issues) => ProjectFeature::ENABLED) - end - - def import_data - { - data: { - phabricator_url: import_url - }, - credentials: { - api_token: params.fetch(:api_token) - } - } - end - end - end -end diff --git a/lib/gitlab/phabricator_import/representation/task.rb b/lib/gitlab/phabricator_import/representation/task.rb deleted file mode 100644 index ba93fb37a8e..00000000000 --- a/lib/gitlab/phabricator_import/representation/task.rb +++ /dev/null @@ -1,72 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - module Representation - class Task - def initialize(json) - @json = json - end - - def phabricator_id - json['phid'] - end - - def author_phid - json['fields']['authorPHID'] - end - - def owner_phid - json['fields']['ownerPHID'] - end - - def phids - @phids ||= [author_phid, owner_phid] - end - - def issue_attributes - @issue_attributes ||= { - title: issue_title, - description: issue_description, - state: issue_state, - created_at: issue_created_at, - closed_at: issue_closed_at - } - end - - private - - attr_reader :json - - def issue_title - # The 255 limit is the validation we impose on the Issue title in - # Issuable - @issue_title ||= json['fields']['name'].truncate(255) - end - - def issue_description - json['fields']['description']['raw'] - end - - def issue_state - issue_closed_at.present? ? :closed : :opened - end - - def issue_created_at - return unless json['fields']['dateCreated'] - - @issue_created_at ||= cast_datetime(json['fields']['dateCreated']) - end - - def issue_closed_at - return unless json['fields']['dateClosed'] - - @issue_closed_at ||= cast_datetime(json['fields']['dateClosed']) - end - - def cast_datetime(value) - Time.at(value.to_i) - end - end - end - end -end diff --git a/lib/gitlab/phabricator_import/representation/user.rb b/lib/gitlab/phabricator_import/representation/user.rb deleted file mode 100644 index 7fd7cecc6ae..00000000000 --- a/lib/gitlab/phabricator_import/representation/user.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - module Representation - class User - def initialize(json) - @json = json - end - - def phabricator_id - json['phid'] - end - - def username - json['fields']['username'] - end - - private - - attr_reader :json - end - end - end -end diff --git a/lib/gitlab/phabricator_import/user_finder.rb b/lib/gitlab/phabricator_import/user_finder.rb deleted file mode 100644 index c6058d12527..00000000000 --- a/lib/gitlab/phabricator_import/user_finder.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module PhabricatorImport - class UserFinder - def initialize(project, phids) - @project = project - @phids = phids - @loaded_phids = Set.new - end - - def find(phid) - found_user = object_map.get_gitlab_model(phid) do - find_user_for_phid(phid) - end - - loaded_phids << phid - - found_user - end - - private - - attr_reader :project, :phids, :loaded_phids - - def object_map - @object_map ||= Gitlab::PhabricatorImport::Cache::Map.new(project) - end - - def find_user_for_phid(phid) - phabricator_user = phabricator_users.find { |u| u.phabricator_id == phid } - return unless phabricator_user - - project.authorized_users.find_by_username(phabricator_user.username) - end - - def phabricator_users - @user_responses ||= client.users(users_to_request).flat_map(&:users) - end - - def users_to_request - phids - loaded_phids.to_a - end - - def client - @client ||= - Gitlab::PhabricatorImport::Conduit::User - .new(phabricator_url: project.import_data.data['phabricator_url'], - api_token: project.import_data.credentials[:api_token]) - end - end - end -end diff --git a/lib/gitlab/phabricator_import/worker_state.rb b/lib/gitlab/phabricator_import/worker_state.rb deleted file mode 100644 index ffa2d3d7a43..00000000000 --- a/lib/gitlab/phabricator_import/worker_state.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true -module Gitlab - module PhabricatorImport - class WorkerState - def initialize(project_id) - @project_id = project_id - end - - def add_job - redis.with do |r| - r.pipelined do |pipe| - pipe.incr(all_jobs_key) - pipe.expire(all_jobs_key, timeout) - end - end - end - - def remove_job - redis.with do |r| - r.decr(all_jobs_key) - end - end - - def running_count - redis.with { |r| r.get(all_jobs_key) }.to_i - end - - private - - attr_reader :project_id - - def redis - Gitlab::Redis::SharedState - end - - def all_jobs_key - @all_jobs_key ||= "phabricator-import/jobs/project-#{project_id}/job-count" - end - - def timeout - # Make sure we get rid of all the information after a job is marked - # as failed/succeeded - Gitlab::Import::StuckImportJob::IMPORT_JOBS_EXPIRATION - end - end - end -end diff --git a/lib/gitlab/private_commit_email.rb b/lib/gitlab/private_commit_email.rb index 886c2c32d36..176402cadfe 100644 --- a/lib/gitlab/private_commit_email.rb +++ b/lib/gitlab/private_commit_email.rb @@ -19,7 +19,7 @@ module Gitlab end def user_ids_for_emails(emails) - emails.map { |email| user_id_for_email(email) }.compact.uniq + emails.filter_map { |email| user_id_for_email(email) }.uniq end def for_user(user) diff --git a/lib/gitlab/project_authorizations.rb b/lib/gitlab/project_authorizations.rb index da3f67dde51..c770260a66e 100644 --- a/lib/gitlab/project_authorizations.rb +++ b/lib/gitlab/project_authorizations.rb @@ -12,7 +12,12 @@ module Gitlab end def calculate - cte = recursive_cte + cte = if Feature.enabled?(:linear_project_authorization, user) + linear_cte + else + recursive_cte + end + cte_alias = cte.table.alias(Group.table_name) projects = Project.arel_table links = ProjectGroupLink.arel_table @@ -47,11 +52,18 @@ module Gitlab .where('p_ns.share_with_group_lock IS FALSE') ] - ProjectAuthorization - .unscoped - .with - .recursive(cte.to_arel) - .select_from_union(relations) + if Feature.enabled?(:linear_project_authorization, user) + ProjectAuthorization + .unscoped + .with(cte.to_arel) + .select_from_union(relations) + else + ProjectAuthorization + .unscoped + .with + .recursive(cte.to_arel) + .select_from_union(relations) + end end private @@ -89,6 +101,30 @@ module Gitlab cte end + def linear_cte + # Groups shared with user and their parent groups + shared_groups = Group + .select("namespaces.id, MAX(LEAST(members.access_level, group_group_links.group_access)) as access_level") + .joins("INNER JOIN group_group_links ON group_group_links.shared_group_id = namespaces.id + OR namespaces.traversal_ids @> ARRAY[group_group_links.shared_group_id::int]") + .joins("INNER JOIN members ON group_group_links.shared_with_group_id = members.source_id") + .merge(user.group_members) + .merge(GroupMember.active_state) + .group("namespaces.id") + + # Groups the user is a member of and their parent groups. + lateral_query = Group.as_ids.where("namespaces.traversal_ids @> ARRAY [members.source_id]") + member_groups_with_ancestors = GroupMember.select("namespaces.id, MAX(members.access_level) as access_level") + .joins("CROSS JOIN LATERAL (#{lateral_query.to_sql}) as namespaces") + .group("namespaces.id") + .merge(user.group_members) + .merge(GroupMember.active_state) + + union = Namespace.from_union([shared_groups, member_groups_with_ancestors]) + + Gitlab::SQL::CTE.new(:linear_namespaces_cte, union) + end + # Builds a LEFT JOIN to join optional memberships onto the CTE. def join_members_on_namespaces members = Member.arel_table diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 5394cd115b1..c9ed4720e83 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -66,7 +66,6 @@ module Gitlab ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), - ProjectTemplate.new('gitbook', 'Pages/GitBook', _('Everything you need to create a GitLab Pages site using GitBook'), 'https://gitlab.com/pages/gitbook', 'illustrations/logos/gitbook.svg'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'), ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), @@ -81,8 +80,9 @@ module Gitlab ProjectTemplate.new('jsonnet', 'Jsonnet for Dynamic Child Pipelines', _('An example showing how to use Jsonnet with GitLab dynamic child pipelines'), 'https://gitlab.com/gitlab-org/project-templates/jsonnet'), ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), - ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg') - ].freeze + ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg'), + ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg') + ] end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/prometheus/internal.rb b/lib/gitlab/prometheus/internal.rb index fe06b97add6..55e1837d101 100644 --- a/lib/gitlab/prometheus/internal.rb +++ b/lib/gitlab/prometheus/internal.rb @@ -27,7 +27,7 @@ module Gitlab def self.server_address Gitlab.config.prometheus.server_address.to_s if Gitlab.config.prometheus - rescue Settingslogic::MissingSetting + rescue GitlabSettings::MissingSetting Gitlab::AppLogger.error('Prometheus server_address is not present in config/gitlab.yml') nil @@ -35,7 +35,7 @@ module Gitlab def self.prometheus_enabled? Gitlab.config.prometheus.enabled if Gitlab.config.prometheus - rescue Settingslogic::MissingSetting + rescue GitlabSettings::MissingSetting Gitlab::AppLogger.error('prometheus.enabled is not present in config/gitlab.yml') false diff --git a/lib/gitlab/prometheus/queries/knative_invocation_query.rb b/lib/gitlab/prometheus/queries/knative_invocation_query.rb deleted file mode 100644 index 6438995b576..00000000000 --- a/lib/gitlab/prometheus/queries/knative_invocation_query.rb +++ /dev/null @@ -1,42 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Prometheus - module Queries - class KnativeInvocationQuery < BaseQuery - include QueryAdditionalMetrics - - def query(serverless_function_id) - PrometheusMetricsFinder - .new(identifier: :system_metrics_knative_function_invocation_count, common: true) - .execute - .first - .to_query_metric - .tap do |q| - q.queries[0][:result] = run_query(q.queries[0][:query_range], context(serverless_function_id)) - end - end - - protected - - def context(function_id) - function = ::Serverless::Function.find_by_id(function_id) - { - function_name: function.name, - kube_namespace: function.namespace - } - end - - def run_query(query, context) - query %= context - client_query_range(query, start_time: 8.hours.ago.to_f, end_time: Time.now.to_f) - end - - def self.transform_reactive_result(result) - result[:metrics] = result.delete :data - result - end - end - end - end -end diff --git a/lib/gitlab/puma_logging/json_formatter.rb b/lib/gitlab/puma_logging/json_formatter.rb index 9eeb980fc53..6d97b6615aa 100644 --- a/lib/gitlab/puma_logging/json_formatter.rb +++ b/lib/gitlab/puma_logging/json_formatter.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'json' +require 'time' module Gitlab module PumaLogging diff --git a/lib/gitlab/quick_actions/extractor.rb b/lib/gitlab/quick_actions/extractor.rb index 2e4817e6b17..015dbe7063c 100644 --- a/lib/gitlab/quick_actions/extractor.rb +++ b/lib/gitlab/quick_actions/extractor.rb @@ -63,11 +63,12 @@ module Gitlab #{CODE_REGEX} | #{INLINE_CODE_REGEX} | #{HTML_BLOCK_REGEX} | #{QUOTE_BLOCK_REGEX} }mix.freeze - attr_reader :command_definitions + attr_reader :command_definitions, :keep_actions - def initialize(command_definitions) + def initialize(command_definitions, keep_actions: false) @command_definitions = command_definitions @commands_regex = {} + @keep_actions = keep_actions end # Extracts commands from content and return an array of commands. @@ -76,8 +77,8 @@ module Gitlab # ['command1'], # ['command3', 'arg1 arg2'], # ] - # The command and the arguments are stripped. - # The original command text is removed from the given `content`. + # The original command text and arguments are removed from the given `content`, + # unless `keep_actions` is true. # # Usage: # ``` @@ -85,6 +86,11 @@ module Gitlab # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] # msg #=> "hello\nworld" + # + # extractor = Gitlab::QuickActions::Extractor.new([:open, :assign, :labels], keep_actions: true) + # msg = %(hello\n/labels ~foo ~"bar baz"\nworld) + # commands = extractor.extract_commands(msg) #=> [['labels', '~foo ~"bar baz"']] + # msg #=> "hello\n/labels ~foo ~"bar baz"\n\nworld" # ``` def extract_commands(content, only: nil) return [content, []] unless content @@ -138,6 +144,10 @@ module Gitlab if redact output = "`/#{matched_text[:cmd]}#{" " + matched_text[:arg] if matched_text[:arg]}`" output += "\n" if matched_text[0].include?("\n") + elsif keep_actions + # requires an additional newline so that when rendered, it appears + # on its own line, rather than all on the same line + output = "\n#{matched_text[0]}\n" end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index ae8bc102f57..d7e9e1a980b 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -255,11 +255,12 @@ module Gitlab execution_message { _('Issue has been promoted to incident') } types Issue condition do - !quick_action_target.incident? && + !quick_action_target.work_item_type&.incident? && current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target) end command :promote_to_incident do - @updates[:issue_type] = "incident" + @updates[:issue_type] = :incident + @updates[:work_item_type] = ::WorkItems::Type.default_by_type(:incident) end desc { _('Add customer relation contacts') } @@ -297,7 +298,7 @@ module Gitlab params '<timeline comment> | <date(YYYY-MM-DD)> <time(HH:MM)>' types Issue condition do - quick_action_target.incident? && + quick_action_target.work_item_type&.incident? && current_user.can?(:admin_incident_management_timeline_event, quick_action_target) end parse_params do |event_params| diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index f0ad6653c5e..c374593bf01 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -102,7 +102,7 @@ module Gitlab (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end command :draft do - @updates[:wip_event] = draft_action_command + @updates[:wip_event] = 'draft' end desc { _('Set the Ready status') } @@ -309,21 +309,11 @@ module Gitlab noun = quick_action_target.to_ability_name.humanize(capitalize: false) if !quick_action_target.draft? _("%{verb} this %{noun} as a draft.") - elsif Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) - _("%{verb} this %{noun} as ready.") else _("No change to this %{noun}'s draft status.") end % { verb: verb, noun: noun } end - def draft_action_command - if Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) - quick_action_target.draft? ? 'ready' : 'draft' - else - 'draft' - end - end - def merge_orchestration_service @merge_orchestration_service ||= ::MergeRequests::MergeOrchestrationService.new(project, current_user) end diff --git a/lib/gitlab/quick_actions/relate_actions.rb b/lib/gitlab/quick_actions/relate_actions.rb index 4c8035f192e..b8cbfdefda1 100644 --- a/lib/gitlab/quick_actions/relate_actions.rb +++ b/lib/gitlab/quick_actions/relate_actions.rb @@ -8,19 +8,27 @@ module Gitlab included do desc { _('Mark this issue as related to another issue') } - explanation do |related_reference| - _('Marks this issue as related to %{issue_ref}.') % { issue_ref: related_reference } + explanation do |target_issues| + _('Marks this issue as related to %{issue_ref}.') % { issue_ref: target_issues.to_sentence } end - execution_message do |related_reference| - _('Marked this issue as related to %{issue_ref}.') % { issue_ref: related_reference } + execution_message do |target_issues| + _('Marked this issue as related to %{issue_ref}.') % { issue_ref: target_issues.to_sentence } end - params '#issue' + params '<#issue | group/project#issue | issue URL>' types Issue - condition do - current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target) + condition { can_relate_issues? } + parse_params { |issues| format_params(issues) } + command :relate do |target_issues| + create_links(target_issues) end - command :relate do |related_reference| - service = IssueLinks::CreateService.new(quick_action_target, current_user, { issuable_references: [related_reference] }) + + private + + def create_links(references, type: 'relates_to') + service = IssueLinks::CreateService.new( + quick_action_target, + current_user, { issuable_references: references, link_type: type } + ) create_issue_link = proc { service.execute } if quick_action_target.persisted? @@ -29,6 +37,14 @@ module Gitlab quick_action_target.run_after_commit(&create_issue_link) end end + + def can_relate_issues? + current_user.can?(:admin_issue_link, quick_action_target) + end + + def format_params(issue_references) + issue_references.split(' ') + end end end end diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb new file mode 100644 index 00000000000..fa43308c9e2 --- /dev/null +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + module WorkItemActions + extend ActiveSupport::Concern + include Gitlab::QuickActions::Dsl + + included do + desc { _('Change work item type') } + explanation do |target_type| + format(_("Converts work item to %{type}. Widgets not supported in new type are removed."), type: target_type) + end + types WorkItem + condition do + quick_action_target&.project&.work_items_mvc_2_feature_flag_enabled? + end + params 'Task | Objective | Key Result | Issue' + command :type do |type_name| + work_item_type = ::WorkItems::Type.find_by_name(type_name) + errors = validate_type(work_item_type) + + if errors.present? + @execution_message[:type] = errors + else + @updates[:issue_type] = work_item_type.base_type + @updates[:work_item_type] = work_item_type + @execution_message[:type] = _('Type changed successfully.') + end + end + end + + private + + def validate_type(type) + return type_error(:not_found) unless type.present? + return type_error(:same_type) if quick_action_target.work_item_type == type + return type_error(:forbidden) unless current_user.can?(:"create_#{type.base_type}", quick_action_target) + + nil + end + + def type_error(reason) + message = { + not_found: 'Provided type is not supported', + same_type: 'Types are the same', + forbidden: 'You have insufficient permissions' + }.freeze + + format(_("Failed to convert this work item: %{reason}."), { reason: message[reason] }) + end + end + end +end diff --git a/lib/gitlab/rack_attack.rb b/lib/gitlab/rack_attack.rb index bedbe9c0bff..d999b706d6c 100644 --- a/lib/gitlab/rack_attack.rb +++ b/lib/gitlab/rack_attack.rb @@ -19,7 +19,7 @@ module Gitlab [429, { 'Content-Type' => 'text/plain' }.merge(throttled_headers), [Gitlab::Throttle.rate_limiting_response_text]] end - rack_attack.cache.store = Gitlab::RackAttack::InstrumentedCacheStore.new + rack_attack.cache.store = Gitlab::RackAttack::Store.new # Configure the throttles configure_throttles(rack_attack) diff --git a/lib/gitlab/rack_attack/instrumented_cache_store.rb b/lib/gitlab/rack_attack/instrumented_cache_store.rb deleted file mode 100644 index d8beb259fba..00000000000 --- a/lib/gitlab/rack_attack/instrumented_cache_store.rb +++ /dev/null @@ -1,33 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RackAttack - # This class is a proxy for all Redis calls made by RackAttack. All - # the calls are instrumented, then redirected to the underlying - # store (in `.store). This class instruments the standard interfaces - # of ActiveRecord::Cache defined in - # https://github.com/rails/rails/blob/v6.0.3.1/activesupport/lib/active_support/cache.rb#L315 - # - # For more information, please see - # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/751 - class InstrumentedCacheStore - NOTIFICATION_CHANNEL = 'redis.rack_attack' - - delegate :silence!, :mute, to: :@upstream_store - - def initialize(upstream_store: ::Gitlab::Redis::RateLimiting.cache_store, notifier: ActiveSupport::Notifications) - @upstream_store = upstream_store - @notifier = notifier - end - - [:fetch, :read, :read_multi, :write_multi, :fetch_multi, :write, :delete, - :exist?, :delete_matched, :increment, :decrement, :cleanup, :clear].each do |interface| - define_method interface do |*args, **k_args, &block| - @notifier.instrument(NOTIFICATION_CHANNEL, operation: interface) do - @upstream_store.public_send(interface, *args, **k_args, &block) # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end - end -end diff --git a/lib/gitlab/rack_attack/store.rb b/lib/gitlab/rack_attack/store.rb new file mode 100644 index 00000000000..e4a1b022c32 --- /dev/null +++ b/lib/gitlab/rack_attack/store.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module RackAttack + class Store + InvalidAmount = Class.new(StandardError) + + # The increment method gets called very often. The implementation below + # aims to minimize the number of Redis calls we make. + def increment(key, amount = 1, options = {}) + # Our code below that prevents calling EXPIRE after every INCR assumes + # we always increment by 1. This is true in Rack::Attack as of v6.6.1. + # This guard should alert us if Rack::Attack changes its behavior in a + # future version. + raise InvalidAmount unless amount == 1 + + with do |redis| + key = namespace(key) + new_value = redis.incr(key) + expires_in = options[:expires_in] + redis.expire(key, expires_in) if new_value == 1 && expires_in + new_value + end + end + + def read(key, _options = {}) + with { |redis| redis.get(namespace(key)) } + end + + def write(key, value, options = {}) + with { |redis| redis.set(namespace(key), value, ex: options[:expires_in]) } + end + + def delete(key, _options = {}) + with { |redis| redis.del(namespace(key)) } + end + + private + + def with(&block) + # rubocop: disable CodeReuse/ActiveRecord + Gitlab::Redis::RateLimiting.with(&block) + # rubocop: enable CodeReuse/ActiveRecord + rescue ::Redis::BaseConnectionError + # Following the example of + # https://github.com/rack/rack-attack/blob/v6.6.1/lib/rack/attack/store_proxy/redis_proxy.rb#L61-L65, + # do not raise an error if we cannot connect to Redis. If + # Redis::RateLimiting is unavailable it should not take the site down. + nil + end + + def namespace(key) + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:#{key}" + end + end + end +end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 2de3c07712f..dc13bb927e6 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -11,13 +11,19 @@ module Gitlab end def clear_cache!(key) + use_pipeline = ::Feature.enabled?(:use_pipeline_over_multikey) + with do |redis| keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.pipelined do |pipeline| - keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } + if use_pipeline + keys.each { |key| pipeline.unlink(key) } + else + keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } + end end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 4d15022cca5..64ca89c6bff 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -10,10 +10,10 @@ module Gitlab ALL_CLASSES = [ Gitlab::Redis::Cache, Gitlab::Redis::DbLoadBalancing, + Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, - Gitlab::Redis::ClusterRateLimiting, Gitlab::Redis::Sessions, Gitlab::Redis::SharedState, Gitlab::Redis::TraceChunks diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 647573e59fe..ba3af3e7a6f 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -2,16 +2,6 @@ module Gitlab module Redis - # Match signature in - # https://github.com/rails/rails/blob/v6.1.7.2/activesupport/lib/active_support/cache/redis_cache_store.rb#L59 - ERROR_HANDLER = ->(method:, returning:, exception:) do - Gitlab::ErrorTracking.log_exception( - exception, - method: method, - returning: returning.inspect - ) - end - class Cache < ::Gitlab::Redis::Wrapper CACHE_NAMESPACE = 'cache:gitlab' @@ -22,8 +12,7 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: default_ttl_seconds, - error_handler: ::Gitlab::Redis::ERROR_HANDLER + expires_in: default_ttl_seconds } end diff --git a/lib/gitlab/redis/cluster_rate_limiting.rb b/lib/gitlab/redis/cluster_rate_limiting.rb deleted file mode 100644 index e9d1e4f0c3f..00000000000 --- a/lib/gitlab/redis/cluster_rate_limiting.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class ClusterRateLimiting < ::Gitlab::Redis::Wrapper - def self.config_fallback - Cache - end - end - end -end diff --git a/lib/gitlab/redis/feature_flag.rb b/lib/gitlab/redis/feature_flag.rb new file mode 100644 index 00000000000..441ff669035 --- /dev/null +++ b/lib/gitlab/redis/feature_flag.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + class FeatureFlag < ::Gitlab::Redis::Wrapper + FeatureFlagStore = Class.new(ActiveSupport::Cache::RedisCacheStore) + + class << self + # The data we store on FeatureFlag is currently stored on Cache. + def config_fallback + Cache + end + + def cache_store + @cache_store ||= FeatureFlagStore.new( + redis: pool, + compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), + namespace: Cache::CACHE_NAMESPACE, + expires_in: 1.hour + ) + end + end + end + end +end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index a102267d52b..9571e2f92e6 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -5,12 +5,6 @@ module Gitlab class MultiStore include Gitlab::Utils::StrongMemoize - class ReadFromPrimaryError < StandardError - def message - 'Value not found on the redis primary store. Read from the redis secondary store successful.' - end - end - class PipelinedDiffError < StandardError def initialize(result_primary, result_secondary) @result_primary = result_primary @@ -32,41 +26,33 @@ module Gitlab attr_reader :primary_store, :secondary_store, :instance_name - FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis primary_store.' + FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.' FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze - # For ENUMERATOR_CACHE_HIT_VALIDATOR and READ_CACHE_HIT_VALIDATOR, - # we define procs to validate cache hit. The only other acceptable value is nil, - # in the case of errors being raised. - # - # If a command has no empty response, set ->(val) { true } - # - # Ref: https://www.rubydoc.info/github/redis/redis-rb/Redis/Commands - # - READ_CACHE_HIT_VALIDATOR = { - exists: ->(val) { val != 0 }, - exists?: ->(val) { val }, - get: ->(val) { !val.nil? }, - hexists: ->(val) { val }, - hget: ->(val) { !val.nil? }, - hgetall: ->(val) { val.is_a?(Hash) && !val.empty? }, - hlen: ->(val) { val != 0 }, - hmget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, - hscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - mapped_hmget: ->(val) { val.is_a?(Hash) && !val.compact.empty? }, - mget: ->(val) { val.is_a?(Array) && !val.compact.empty? }, - scan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - scard: ->(val) { val != 0 }, - sismember: ->(val) { val }, - smembers: ->(val) { val.is_a?(Array) && !val.empty? }, - sscan: ->(val) { val != ['0', []] }, - sscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? }, - ttl: ->(val) { val != 0 && val != -2 }, # ttl returns -2 if the key does not exist. See https://redis.io/commands/ttl/ - zscan_each: ->(val) { val.is_a?(Enumerator) && !val.first.nil? } - }.freeze + READ_COMMANDS = %i[ + exists + exists? + get + hexists + hget + hgetall + hlen + hmget + hscan_each + mapped_hmget + mget + scan_each + scard + sismember + smembers + sscan + sscan_each + ttl + zscan_each + ].freeze WRITE_COMMANDS = %i[ del @@ -111,7 +97,7 @@ module Gitlab end # rubocop:disable GitlabSecurity/PublicSend - READ_CACHE_HIT_VALIDATOR.each_key do |name| + READ_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| if use_primary_and_secondary_stores? read_command(name, *args, **kwargs, &block) @@ -186,12 +172,6 @@ module Gitlab @pipelined_command_error.increment(command: command_name, instance_name: instance_name) end - def increment_read_fallback_count(command_name) - @read_fallback_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_read_fallback_total, - 'Client side Redis MultiStore reading fallback') - @read_fallback_counter.increment(command: command_name, instance_name: instance_name) - end - def increment_method_missing_count(command_name) @method_missing_counter ||= Gitlab::Metrics.counter(:gitlab_redis_multi_store_method_missing_total, 'Client side Redis MultiStore method missing') @@ -247,7 +227,7 @@ module Gitlab if @instance send_command(@instance, command_name, *args, **kwargs, &block) else - read_one_with_fallback(command_name, *args, **kwargs, &block) + read_from_default(command_name, *args, **kwargs, &block) end end @@ -259,35 +239,12 @@ module Gitlab end end - def read_one_with_fallback(command_name, *args, **kwargs, &block) - begin - value = send_command(default_store, command_name, *args, **kwargs, &block) - rescue StandardError => e - log_error(e, command_name, - multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) - end - - return value if block.nil? && cache_hit?(command_name, value) - - fallback_read(command_name, *args, **kwargs, &block) - end - - def cache_hit?(command, value) - validator = READ_CACHE_HIT_VALIDATOR[command] - return false unless validator - - !value.nil? && validator.call(value) - end - - def fallback_read(command_name, *args, **kwargs, &block) - value = send_command(fallback_store, command_name, *args, **kwargs, &block) - - if value - log_error(ReadFromPrimaryError.new, command_name) - increment_read_fallback_count(command_name) - end - - value + def read_from_default(command_name, *args, **kwargs, &block) + send_command(default_store, command_name, *args, **kwargs, &block) + rescue StandardError => e + log_error(e, command_name, + multi_store_error_message: FAILED_TO_READ_ERROR_MESSAGE) + raise end def write_both(command_name, *args, **kwargs, &block) diff --git a/lib/gitlab/redis/rate_limiting.rb b/lib/gitlab/redis/rate_limiting.rb index 12710bafbea..74b4ca12d18 100644 --- a/lib/gitlab/redis/rate_limiting.rb +++ b/lib/gitlab/redis/rate_limiting.rb @@ -3,6 +3,9 @@ module Gitlab module Redis class RateLimiting < ::Gitlab::Redis::Wrapper + # We create a subclass only for the purpose of differentiating between different stores in cache metrics + RateLimitingStore = Class.new(ActiveSupport::Cache::RedisCacheStore) + class << self # The data we store on RateLimiting used to be stored on Cache. def config_fallback @@ -10,20 +13,7 @@ module Gitlab end def cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( - redis: pool, - namespace: Cache::CACHE_NAMESPACE, - error_handler: ::Gitlab::Redis::ERROR_HANDLER - ) - end - - private - - def redis - primary_store = ::Redis.new(::Gitlab::Redis::ClusterRateLimiting.params) - secondary_store = ::Redis.new(params) - - MultiStore.new(primary_store, secondary_store, name.demodulize) + @cache_store ||= RateLimitingStore.new(redis: pool, namespace: Cache::CACHE_NAMESPACE) end end end diff --git a/lib/gitlab/redis/repository_cache.rb b/lib/gitlab/redis/repository_cache.rb index 6c7bc8c41d5..966c6584aa5 100644 --- a/lib/gitlab/redis/repository_cache.rb +++ b/lib/gitlab/redis/repository_cache.rb @@ -3,6 +3,9 @@ module Gitlab module Redis class RepositoryCache < ::Gitlab::Redis::Wrapper + # We create a subclass only for the purpose of differentiating between different stores in cache metrics + RepositoryCacheStore = Class.new(ActiveSupport::Cache::RedisCacheStore) + class << self # The data we store on RepositoryCache used to be stored on Cache. def config_fallback @@ -10,12 +13,11 @@ module Gitlab end def cache_store - @cache_store ||= ActiveSupport::Cache::RedisCacheStore.new( + @cache_store ||= RepositoryCacheStore.new( redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: Cache::CACHE_NAMESPACE, - expires_in: Cache.default_ttl_seconds, - error_handler: ::Gitlab::Redis::ERROR_HANDLER + expires_in: Cache.default_ttl_seconds ) end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index c990655769c..45fe04835cc 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -55,15 +55,11 @@ module Gitlab def config_file_name [ # Instance specific config sources: - ENV["GITLAB_REDIS_#{store_name.underscore.upcase}_CONFIG_FILE"], config_file_path("redis.#{store_name.underscore}.yml"), # The current Redis instance may have been split off from another one # (e.g. TraceChunks was split off from SharedState). - config_fallback&.config_file_name, - - # Global config sources: - ENV['GITLAB_REDIS_CONFIG_FILE'] + config_fallback&.config_file_name ].compact.first end @@ -160,35 +156,10 @@ module Gitlab def raw_config_hash config_data = fetch_config - config_hash = - if config_data - config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys - else - { url: '' } - end - - if config_hash[:url].blank? && config_hash[:cluster].blank? - config_hash[:url] = legacy_fallback_urls[self.class.store_name] || legacy_fallback_urls[self.class.config_fallback.store_name] - end - - config_hash - end + return { url: '' } if config_data.nil? + return { url: config_data } if config_data.is_a?(String) - # These URLs were defined for cache, queues, and shared_state in - # code. They are used only when no config file exists at all for a - # given instance. The configuration does not seem particularly - # useful - it uses different ports on localhost - but we cannot - # confidently delete it as we don't know if any instances rely on - # this. - # - # DO NOT ADD new instances here. All new instances should define a - # `.config_fallback`, which will then be used to look up this URL. - def legacy_fallback_urls - { - 'Cache' => 'redis://localhost:6380', - 'Queues' => 'redis://localhost:6381', - 'SharedState' => 'redis://localhost:6382' - } + config_data.deep_symbolize_keys end def fetch_config diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index 540394f04bd..783b68fac12 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -4,7 +4,7 @@ module Gitlab # Extract possible GFM references from an arbitrary String for further processing. class ReferenceExtractor < Banzai::ReferenceExtractor REFERABLES = %i(user issue label milestone mentioned_user mentioned_group mentioned_project - merge_request snippet commit commit_range directly_addressed_user epic iteration vulnerability + merge_request snippet commit commit_range directly_addressed_user epic vulnerability alert).freeze attr_accessor :project, :current_user, :author @@ -64,18 +64,24 @@ module Gitlab end def all - REFERABLES.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend + self.class.referrables.each { |referable| send(referable.to_s.pluralize) } # rubocop:disable GitlabSecurity/PublicSend @references.values.flatten end - def self.references_pattern - return @pattern if @pattern + class << self + def references_pattern + return @pattern if @pattern - patterns = REFERABLES.map do |type| - Banzai::ReferenceParser[type].reference_class.try(:reference_pattern) - end.uniq + patterns = referrables.map do |type| + Banzai::ReferenceParser[type].reference_class.try(:reference_pattern) + end.uniq - @pattern = Regexp.union(patterns.compact) + @pattern = Regexp.union(patterns.compact) + end + + def referrables + @referrables ||= REFERABLES + end end private @@ -90,3 +96,5 @@ module Gitlab end end end + +Gitlab::ReferenceExtractor.prepend_mod diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 943218a9972..eb99805e2e8 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,7 +5,12 @@ module Gitlab module Packages CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze + PYPI_NORMALIZED_NAME_REGEX_STRING = '[-_.]+' + + # see https://github.com/apache/maven/blob/c1dfb947b509e195c75d4275a113598cf3063c3e/maven-artifact/src/main/java/org/apache/maven/artifact/Artifact.java#L46 + MAVEN_SNAPSHOT_DYNAMIC_PARTS = /\A.{0,1000}(-\d{8}\.\d{6}-\d+).{0,1000}\z/.freeze + API_PATH_REGEX = %r{^/api/v\d+/(projects/[^/]+/|groups?/[^/]+/-/)?packages/[A-Za-z]+}.freeze def conan_package_reference_regex @@ -141,7 +146,7 @@ module Gitlab end def debian_direct_upload_filename_regex - @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb)\z}o.freeze + @debian_direct_upload_filename_regex ||= %r{\A.*\.(deb|udeb|ddeb)\z}o.freeze end def helm_channel_regex @@ -253,38 +258,45 @@ module Gitlab end end - extend self - extend Packages + module BulkImports + def bulk_import_destination_namespace_path_regex + # This regexp validates the string conforms to rules for a destination_namespace path: + # i.e does not start with a non-alphanumeric character, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' + # the regex also allows for an empty string ('') to be accepted as this is allowed in + # a bulk_import POST request + @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|(\A[0-9a-z]*(-_.)?[0-9a-z])(\/?[0-9a-z]*[-_.]?[0-9a-z])+\z)/i + end - def bulk_import_destination_namespace_path_regex - # This regexp validates the string conforms to rules for a destination_namespace path: - # i.e does not start with a non-alphanumeric character except for periods or underscores, - # contains only alphanumeric characters, forward slashes, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/destination/namespace/path' - # the regex also allows for an empty string ('') to be accepted as this is allowed in - # a bulk_import POST request - @bulk_import_destination_namespace_path_regex ||= %r/((\A\z)|\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z)/i - end + def bulk_import_source_full_path_regex + # This regexp validates the string conforms to rules for a source_full_path path: + # i.e does not start with a non-alphanumeric character except for periods or underscores, + # contains only alphanumeric characters, forward slashes, periods, and underscores, + # does not end with a period or forward slash, and has a relative path structure + # with no http protocol chars or leading or trailing forward slashes + # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' + @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?([-_.+]*)*[0-9a-z][-_]*)+\z/i + end - def bulk_import_source_full_path_regex - # This regexp validates the string conforms to rules for a source_full_path path: - # i.e does not start with a non-alphanumeric character except for periods or underscores, - # contains only alphanumeric characters, forward slashes, periods, and underscores, - # does not end with a period or forward slash, and has a relative path structure - # with no http protocol chars or leading or trailing forward slashes - # eg 'source/full/path' or 'destination_namespace' not 'https://example.com/source/full/path' - @bulk_import_source_full_path_regex ||= %r/\A([.]?)[^\W](\/?[.]?[0-9a-z][-_]*)+\z/i - end + def bulk_import_source_full_path_regex_message + bulk_import_destination_namespace_path_regex_message + end - def bulk_import_destination_namespace_path_regex_message - "cannot start with a non-alphanumeric character except for periods or underscores, " \ - "can contain only alphanumeric characters, forward slashes, periods, and underscores, " \ - "cannot end with a period or forward slash, and has a relative path structure " \ - "with no http protocol chars or leading or trailing forward slashes" \ + def bulk_import_destination_namespace_path_regex_message + "must have a relative path structure " \ + "with no HTTP protocol characters, or leading or trailing forward slashes. " \ + "Path segments must not start or end with a special character, " \ + "and must not contain consecutive special characters." + end end + extend self + extend Packages + extend BulkImports + def group_path_regex # This regexp validates the string conforms to rules for a group slug: # i.e does not start with a non-alphanumeric character except for periods or underscores, @@ -297,7 +309,7 @@ module Gitlab def group_path_regex_message "cannot start with a non-alphanumeric character except for periods or underscores, " \ "can contain only alphanumeric characters, periods, and underscores, " \ - "cannot end with a period or forward slash, and has no leading or trailing forward slashes" \ + "cannot end with a period or forward slash, and has no leading or trailing forward slashes." \ end def project_name_regex @@ -454,7 +466,7 @@ module Gitlab # ``` MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED = '(?P<code>' \ - '^```\n' \ + '^```.*?\n' \ '(?:\n|.)*?' \ '\n```\ *$' \ ')'.freeze @@ -472,6 +484,17 @@ module Gitlab ) }mx.freeze + # HTML block: + # <tag> + # Anything, including `>>>` blocks which are ignored by this filter + # </tag> + MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED = + '(?P<html>' \ + '^<[^>]+?>\ *\n' \ + '(?:\n|.)*?' \ + '\n<\/[^>]+?>\ *$' \ + ')'.freeze + # HTML comment line: # <!-- some commented text --> MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED = @@ -494,6 +517,13 @@ module Gitlab }mx.freeze end + def markdown_code_or_html_blocks_untrusted + @markdown_code_or_html_blocks_untrusted ||= + "#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \ + "|" \ + "#{MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED}" + end + def markdown_code_or_html_comments_untrusted @markdown_code_or_html_comments_untrusted ||= "#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \ @@ -503,6 +533,17 @@ module Gitlab "#{MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED}" end + def markdown_code_or_html_blocks_or_html_comments_untrusted + @markdown_code_or_html_comments_untrusted ||= + "#{MARKDOWN_CODE_BLOCK_REGEX_UNTRUSTED}" \ + "|" \ + "#{MARKDOWN_HTML_BLOCK_REGEX_UNTRUSTED}" \ + "|" \ + "#{MARKDOWN_HTML_COMMENT_LINE_REGEX_UNTRUSTED}" \ + "|" \ + "#{MARKDOWN_HTML_COMMENT_BLOCK_REGEX_UNTRUSTED}" + end + # Based on Jira's project key format # https://confluence.atlassian.com/adminjiraserver073/changing-the-project-key-format-861253229.html # Avoids linking CVE IDs (https://cve.mitre.org/cve/identifiers/syntaxchange.html#new) as Jira issues. @@ -550,11 +591,15 @@ module Gitlab end def issue - @issue ||= /(?<issue>\d+)(?<format>\+)?(?=\W|\z)/ + @issue ||= /(?<issue>\d+)(?<format>\+s{,1})?(?=\W|\z)/ + end + + def work_item + @work_item ||= /(?<work_item>\d+)(?<format>\+s{,1})?(?=\W|\z)/ end def merge_request - @merge_request ||= /(?<merge_request>\d+)(?<format>\+)?/ + @merge_request ||= /(?<merge_request>\d+)(?<format>\+s{,1})?/ end def base64_regex diff --git a/lib/gitlab/registration_features/password_complexity.rb b/lib/gitlab/registration_features/password_complexity.rb new file mode 100644 index 00000000000..6d165a7a665 --- /dev/null +++ b/lib/gitlab/registration_features/password_complexity.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module RegistrationFeatures + class PasswordComplexity + def self.feature_available? + ::License.feature_available?(:password_complexity) || + ::GitlabSubscriptions::Features.usage_ping_feature?(:password_complexity) + end + end + end +end diff --git a/lib/gitlab/repository_size_error_message.rb b/lib/gitlab/repository_size_error_message.rb index f5d82e61187..e7d527dd4ce 100644 --- a/lib/gitlab/repository_size_error_message.rb +++ b/lib/gitlab/repository_size_error_message.rb @@ -7,7 +7,8 @@ module Gitlab delegate :current_size, :limit, :exceeded_size, :additional_repo_storage_available?, to: :@checker # @param checker [RepositorySizeChecker] - def initialize(checker) + def initialize(checker, message_params = {}) + @message_params = message_params @checker = checker end @@ -19,13 +20,21 @@ module Gitlab "This merge request cannot be merged, #{base_message}" end + def push_warning + _("##### WARNING ##### You have used %{usage_percentage} of the storage quota for %{namespace_name} " \ + "(%{current_size} of %{size_limit}). If %{namespace_name} exceeds the storage quota, " \ + "all projects in the namespace will be locked and actions will be restricted. " \ + "To manage storage, or purchase additional storage, see %{manage_storage_url}. " \ + "To learn more about restricted actions, see %{restricted_actions_url}") % push_message_params + end + def push_error(change_size = 0) "Your push has been rejected, #{base_message(change_size)}. #{more_info_message}" end def new_changes_error if additional_repo_storage_available? - "Your push to this repository has been rejected because it would exceed storage limits. Please contact your GitLab administrator for more information." + "Your push to this repository has been rejected because it would exceed storage limits. #{more_info_message}" else "Your push to this repository would cause it to exceed the size limit of #{formatted(limit)} so it has been rejected. #{more_info_message}" end @@ -41,6 +50,19 @@ module Gitlab private + attr_reader :message_params + + def push_message_params + { + namespace_name: message_params[:namespace_name], + manage_storage_url: help_page_url('user/usage_quotas', 'manage-your-storage-usage'), + restricted_actions_url: help_page_url('user/read_only_namespaces', 'restricted-actions'), + current_size: formatted(current_size), + size_limit: formatted(limit), + usage_percentage: usage_percentage + } + end + def base_message(change_size = 0) "because this repository has exceeded its size limit of #{formatted(limit)} by #{formatted(exceeded_size(change_size))}" end @@ -48,5 +70,13 @@ module Gitlab def formatted(number) number_to_human_size(number, delimiter: ',', precision: 2) end + + def usage_percentage + number_to_percentage(@checker.usage_ratio * 100, precision: 0) + end + + def help_page_url(path, anchor = nil) + ::Gitlab::Routing.url_helpers.help_page_url(path, anchor: anchor) + end end end diff --git a/lib/gitlab/request_context.rb b/lib/gitlab/request_context.rb index c9eefe9a647..813468ece90 100644 --- a/lib/gitlab/request_context.rb +++ b/lib/gitlab/request_context.rb @@ -7,12 +7,28 @@ module Gitlab RequestDeadlineExceeded = Class.new(StandardError) - attr_accessor :client_ip, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations + attr_accessor :client_ip, :spam_params, :start_thread_cpu_time, :request_start_time, :thread_memory_allocations class << self def instance Gitlab::SafeRequestStore[:request_context] ||= new end + + def start_request_context(request:) + # We need to use Rack::Request to be consistent with Rails due to a Rails bug described in + # https://gitlab.com/gitlab-org/gitlab-foss/issues/58573#note_149799010 + # Hosts behind a load balancer will only see 127.0.0.1 for the load balancer's IP. + rack_req = Rack::Request.new(request.env) + instance.client_ip = rack_req.ip + + instance.spam_params = ::Spam::SpamParams.new_from_request(request: request) + instance.request_start_time = Gitlab::Metrics::System.real_time + end + + def start_thread_context + instance.start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + instance.thread_memory_allocations = Gitlab::Memory::Instrumentation.start_thread_memory_allocations + end end def request_deadline diff --git a/lib/gitlab/resource_events/assignment_event_recorder.rb b/lib/gitlab/resource_events/assignment_event_recorder.rb new file mode 100644 index 00000000000..94bd05a17ba --- /dev/null +++ b/lib/gitlab/resource_events/assignment_event_recorder.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module ResourceEvents + class AssignmentEventRecorder + BATCH_SIZE = 100 + + def initialize(parent:, old_assignees:) + @parent = parent + @old_assignees = old_assignees + end + + def record + return if Feature.disabled?(:record_issue_and_mr_assignee_events, parent.project) + + case parent + when Issue + record_for_parent( + ::ResourceEvents::IssueAssignmentEvent, + :issue_id, + parent, + old_assignees + ) + when MergeRequest + record_for_parent( + ::ResourceEvents::MergeRequestAssignmentEvent, + :merge_request_id, + parent, + old_assignees + ) + end + end + + private + + attr_reader :parent, :old_assignees + + def record_for_parent(resource_klass, foreign_key, parent, old_assignees) + removed_events = (old_assignees - parent.assignees).map do |unassigned_user| + { + foreign_key => parent.id, + user_id: unassigned_user.id, + action: :remove + } + end.to_set + + added_events = (parent.assignees.to_a - old_assignees).map do |added_user| + { + foreign_key => parent.id, + user_id: added_user.id, + action: :add + } + end.to_set + + (removed_events + added_events).each_slice(BATCH_SIZE) do |events| + resource_klass.insert_all(events) + end + end + end + end +end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 7e9fb82fb8b..f74f1489405 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -38,7 +38,7 @@ module Gitlab end def puma? - !!defined?(::Puma) + !!defined?(::Puma::Server) end def sidekiq? diff --git a/lib/gitlab/saas.rb b/lib/gitlab/saas.rb index 16a7a697e6a..722475ce61d 100644 --- a/lib/gitlab/saas.rb +++ b/lib/gitlab/saas.rb @@ -49,6 +49,10 @@ module Gitlab "https://about.gitlab.com/pricing#faq" end + def self.about_feature_comparison_url + "https://about.gitlab.com/pricing/gitlab-com/feature-comparison" + end + def self.doc_url 'https://docs.gitlab.com' end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index 37414f9e2b1..93befc2df57 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -120,6 +120,14 @@ module Gitlab [] end + def failed? + false + end + + def error + nil + end + private def collection_for(scope) diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb index 082d267442c..23043fc8652 100644 --- a/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_seeder.rb @@ -66,14 +66,18 @@ module Gitlab plan_limits = Plan.default.actual_limits if plan_limits.ci_registered_group_runners < @runner_count - logger.error('The plan limits for group runners is set to ' \ + warn 'The plan limits for group runners is set to ' \ "#{plan_limits.ci_registered_group_runners} runners. " \ - 'You should raise the plan limits to avoid errors during runner creation') + "You should raise the plan limits to avoid errors during runner creation by running " \ + "the following command in the Rails console:\n" \ + "Plan.default.actual_limits.update!(ci_registered_group_runners: #{@runner_count})" return false elsif plan_limits.ci_registered_project_runners < @runner_count - logger.error('The plan limits for project runners is set to ' \ + warn 'The plan limits for project runners is set to ' \ "#{plan_limits.ci_registered_project_runners} runners. " \ - 'You should raise the plan limits to avoid errors during runner creation') + "You should raise the plan limits to avoid errors during runner creation by running " \ + "the following command in the Rails console:\n" \ + "Plan.default.actual_limits.update!(ci_registered_project_runners: #{@runner_count})" return false end @@ -205,7 +209,7 @@ module Gitlab scope.runners_token end - response = ::Ci::Runners::RegisterRunnerService.new.execute(runners_token, name: name, **args) + response = ::Ci::Runners::RegisterRunnerService.new(runners_token, name: name, **args).execute runner = response.payload[:runner] ::Ci::Runners::ProcessRunnerVersionUpdateWorker.new.perform(args[:version]) diff --git a/lib/gitlab/seeders/ci/variables_group_seeder.rb b/lib/gitlab/seeders/ci/variables_group_seeder.rb new file mode 100644 index 00000000000..e5c5ee4b75f --- /dev/null +++ b/lib/gitlab/seeders/ci/variables_group_seeder.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + class VariablesGroupSeeder + DEFAULT_SEED_COUNT = 10 + DEFAULT_PREFIX = 'GROUP_VAR_' + DEFAULT_ENV = '*' + + def initialize(params) + @group = Group.find_by_name(params[:name]) + @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT + @environment_scope = params[:environment_scope] || DEFAULT_ENV + @prefix = params[:prefix] || DEFAULT_PREFIX + end + + def seed + if @group.nil? + warn 'ERROR: Group name is invalid.' + return + end + + max_id = group.variables.maximum(:id).to_i + seed_count.times do + max_id += 1 + create_ci_variable(max_id) + end + end + + private + + attr_reader :environment_scope, :group, :prefix, :seed_count + + def create_ci_variable(id) + env = environment_scope == 'unique' ? "env_#{id}" : environment_scope + key = "#{prefix}#{id}" + + if group.variables.by_environment_scope(env).find_by_key(key).present? + warn "WARNING: Group CI Variable with key '#{key}' already exists. Skipping to next CI variable..." + return + end + + group.variables.create( + environment_scope: env, + key: key, + value: SecureRandom.hex(32) + ) + end + end + end + end +end diff --git a/lib/gitlab/seeders/ci/variables_instance_seeder.rb b/lib/gitlab/seeders/ci/variables_instance_seeder.rb new file mode 100644 index 00000000000..d43defd192f --- /dev/null +++ b/lib/gitlab/seeders/ci/variables_instance_seeder.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + class VariablesInstanceSeeder + DEFAULT_SEED_COUNT = 10 + DEFAULT_PREFIX = 'INSTANCE_VAR_' + + def initialize(params = {}) + @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT + @prefix = params[:prefix] || DEFAULT_PREFIX + end + + def seed + max_id = ::Ci::InstanceVariable.maximum(:id).to_i + seed_count.times do + max_id += 1 + create_ci_variable(max_id) + end + end + + private + + attr_reader :prefix, :seed_count + + def create_ci_variable(id) + key = "#{prefix}#{id}" + + if ::Ci::InstanceVariable.find_by_key(key) + warn "WARNING: Instance CI Variable with key '#{key}' already exists. Skipping to next CI variable..." + return + end + + ::Ci::InstanceVariable.new( + key: key, + value: SecureRandom.hex(32) + ).save + end + end + end + end +end diff --git a/lib/gitlab/seeders/ci/variables_project_seeder.rb b/lib/gitlab/seeders/ci/variables_project_seeder.rb new file mode 100644 index 00000000000..c6b3dac7a4d --- /dev/null +++ b/lib/gitlab/seeders/ci/variables_project_seeder.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + class VariablesProjectSeeder + DEFAULT_SEED_COUNT = 10 + DEFAULT_PREFIX = 'VAR_' + DEFAULT_ENV = '*' + + def initialize(params) + @project = Project.find_by_full_path(params[:project_path]) + @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT + @environment_scope = params[:environment_scope] || DEFAULT_ENV + @prefix = params[:prefix] || DEFAULT_PREFIX + end + + def seed + if @project.nil? + warn 'ERROR: Project path is invalid.' + return + end + + max_id = project.variables.maximum(:id).to_i + seed_count.times do + max_id += 1 + create_ci_variable(max_id) + end + end + + private + + attr_reader :environment_scope, :prefix, :project, :seed_count + + def create_ci_variable(id) + env = environment_scope == 'unique' ? "env_#{id}" : environment_scope + key = "#{prefix}#{id}" + + if project.variables.by_environment_scope(env).find_by_key(key).present? + warn "WARNING: Project CI Variable with key '#{key}' already exists. Skipping to next CI variable..." + end + + project.variables.create( + environment_scope: env, + key: key, + value: SecureRandom.hex(32) + ) + end + end + end + end +end diff --git a/lib/gitlab/seeders/project_environment_seeder.rb b/lib/gitlab/seeders/project_environment_seeder.rb new file mode 100644 index 00000000000..3fc7d3d9b12 --- /dev/null +++ b/lib/gitlab/seeders/project_environment_seeder.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + class ProjectEnvironmentSeeder + DEFAULT_SEED_COUNT = 10 + DEFAULT_PREFIX = 'ENV_' + + def initialize(params) + @project = Project.find_by_full_path(params[:project_path]) + @seed_count = params[:seed_count] || DEFAULT_SEED_COUNT + @prefix = params[:prefix] || DEFAULT_PREFIX + end + + def seed + if @project.nil? + warn 'ERROR: Project path is invalid.' + return + end + + max_id = project.environments.maximum(:id).to_i + seed_count.times do + max_id += 1 + create_project_environment_scope(max_id) + end + end + + private + + attr_reader :project, :seed_count, :prefix + + def create_project_environment_scope(id) + name = "#{prefix}#{id}" + + if project.environments.find_by_name(name).present? + warn "WARNING: Project Environment '#{name}' already exists. Skipping to next CI variable..." + return + end + + project.environments.create(name: name) + end + end + end +end diff --git a/lib/gitlab/serializer/ci/variables.rb b/lib/gitlab/serializer/ci/variables.rb index 9abf3a54f37..a12bda0e5a7 100644 --- a/lib/gitlab/serializer/ci/variables.rb +++ b/lib/gitlab/serializer/ci/variables.rb @@ -12,7 +12,7 @@ module Gitlab def load(string) return unless string - object = YAML.safe_load(string, [Symbol]) + object = YAML.safe_load(string, permitted_classes: [Symbol]) object.map do |variable| variable.symbolize_keys.tap do |variable| diff --git a/lib/gitlab/serverless/service.rb b/lib/gitlab/serverless/service.rb deleted file mode 100644 index c3ab2e9ddeb..00000000000 --- a/lib/gitlab/serverless/service.rb +++ /dev/null @@ -1,102 +0,0 @@ -# frozen_string_literal: true - -class Gitlab::Serverless::Service - include Gitlab::Utils::StrongMemoize - - def initialize(attributes) - @attributes = attributes - end - - def name - @attributes.dig('metadata', 'name') - end - - def namespace - @attributes.dig('metadata', 'namespace') - end - - def environment_scope - @attributes.dig('environment_scope') - end - - def environment - @attributes.dig('environment') - end - - def podcount - @attributes.dig('podcount') - end - - def created_at - strong_memoize(:created_at) do - timestamp = @attributes.dig('metadata', 'creationTimestamp') - DateTime.parse(timestamp) if timestamp - end - end - - def image - @attributes.dig( - 'spec', - 'runLatest', - 'configuration', - 'build', - 'template', - 'name') - end - - def description - knative_07_description || knative_05_06_description - end - - def cluster - @attributes.dig('cluster') - end - - def url - proxy_url || knative_06_07_url || knative_05_url - end - - private - - def proxy_url - if cluster&.serverless_domain - ::Serverless::Domain.new( - function_name: name, - serverless_domain_cluster: cluster.serverless_domain, - environment: environment - ).uri.to_s - end - end - - def knative_07_description - @attributes.dig( - 'spec', - 'template', - 'metadata', - 'annotations', - 'Description' - ) - end - - def knative_05_06_description - @attributes.dig( - 'spec', - 'runLatest', - 'configuration', - 'revisionTemplate', - 'metadata', - 'annotations', - 'Description') - end - - def knative_05_url - domain = @attributes.dig('status', 'domain') - return unless domain - - "http://#{domain}" - end - - def knative_06_07_url - @attributes.dig('status', 'url') - end -end diff --git a/lib/gitlab/service_desk.rb b/lib/gitlab/service_desk.rb index b3d6e890e03..5acbde552c8 100644 --- a/lib/gitlab/service_desk.rb +++ b/lib/gitlab/service_desk.rb @@ -10,7 +10,7 @@ module Gitlab end def self.supported? - Gitlab::IncomingEmail.enabled? && Gitlab::IncomingEmail.supports_wildcard? + Gitlab::Email::IncomingEmail.enabled? && Gitlab::Email::IncomingEmail.supports_wildcard? end end end diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb deleted file mode 100644 index bc49efafdda..00000000000 --- a/lib/gitlab/service_desk_email.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ServiceDeskEmail - class << self - include Gitlab::Email::Common - - def config - Gitlab.config.service_desk_email - end - - def key_from_address(address) - wildcard_address = config&.address - return unless wildcard_address - - Gitlab::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address) - end - - def address_for_key(key) - return if config.address.blank? - - config.address.sub(WILDCARD_PLACEHOLDER, key) - end - end - end -end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 3d2ff5a68d2..623b254c4e0 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -22,7 +22,13 @@ module Gitlab keys_to_expire = keys.map { |key| cache_key(key) } Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.unlink(*keys_to_expire) + if ::Feature.enabled?(:use_pipeline_over_multikey) + redis.pipelined do |pipeline| + keys_to_expire.each { |key| pipeline.unlink(key) } + end.sum + else + redis.unlink(*keys_to_expire) + end end end end diff --git a/lib/gitlab/setup_helper.rb b/lib/gitlab/setup_helper.rb index 1e42003b203..2e09a4fce12 100644 --- a/lib/gitlab/setup_helper.rb +++ b/lib/gitlab/setup_helper.rb @@ -80,7 +80,7 @@ module Gitlab # because it uses a Unix socket. # For development and testing purposes, an extra storage is added to gitaly, # which is not known to Rails, but must be explicitly stubbed. - def configuration_toml(gitaly_dir, storage_paths, options, gitaly_ruby: true) + def configuration_toml(gitaly_dir, storage_paths, options) storages = [] address = nil @@ -128,7 +128,6 @@ module Gitlab FileUtils.mkdir(runtime_dir) unless File.exist?(runtime_dir) config[:runtime_dir] = runtime_dir - config[:'gitaly-ruby'] = { dir: File.join(gitaly_dir, 'ruby') } if gitaly_ruby config[:'gitlab-shell'] = { dir: Gitlab.config.gitlab_shell.path } config[:bin_dir] = File.expand_path(File.join(gitaly_dir, '_build', 'bin')) # binaries by default are in `_build/bin` config[:gitlab] = { url: Gitlab.config.gitlab.url } diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 7e2a934b3dd..33a15d95d22 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -57,13 +57,12 @@ module Gitlab @cron_jobs ||= begin Gitlab.config.load_dynamic_cron_schedules! - # Load recurring jobs from gitlab.yml - # UGLY Hack to get nested hash from settingslogic - jobs = Gitlab::Json.parse(Gitlab.config.cron_jobs.to_json) + jobs = Gitlab.config.cron_jobs.to_hash jobs.delete('poll_interval') # Would be interpreted as a job otherwise - # UGLY hack: Settingslogic doesn't allow 'class' key + # Settingslogic (former gem used for yaml configuration) didn't allow 'class' key + # Therefore, we configure cron jobs with `job_class` as a workaround. required_keys = %w[job_class cron] jobs.each do |k, v| if jobs[k] && required_keys.all? { |s| jobs[k].key?(s) } diff --git a/lib/gitlab/sidekiq_config/worker_router.rb b/lib/gitlab/sidekiq_config/worker_router.rb index 0670e5521df..6d5ecb64065 100644 --- a/lib/gitlab/sidekiq_config/worker_router.rb +++ b/lib/gitlab/sidekiq_config/worker_router.rb @@ -77,6 +77,11 @@ module Gitlab def parse_routing_rules(routing_rules) raise InvalidRoutingRuleError, 'The set of routing rule must be an array' unless routing_rules.is_a?(Array) + unless routing_rules.last&.first == WorkerMatcher::WILDCARD_MATCH + Gitlab::AppLogger.warn "sidekiq.routing_rules config is missing a catch-all `*` entry as the last rule. " \ + "Consider adding `[['*', 'default']]` at the end of routing_rules." + end + routing_rules.map do |rule_tuple| raise InvalidRoutingRuleError, "Routing rule `#{rule_tuple.inspect}` is invalid" unless valid_routing_rule?(rule_tuple) diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb deleted file mode 100644 index 1682d62d782..00000000000 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ /dev/null @@ -1,293 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqDaemon - class MemoryKiller < Daemon - include ::Gitlab::Utils::StrongMemoize - - # Today 64-bit CPU support max 256T memory. It is big enough. - MAX_MEMORY_KB = 256 * 1024 * 1024 * 1024 - # RSS below `soft_limit_rss` is considered safe - SOFT_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_MAX_RSS', 2000000).to_i - # RSS above `hard_limit_rss` will be stopped - HARD_LIMIT_RSS_KB = ENV.fetch('SIDEKIQ_MEMORY_KILLER_HARD_LIMIT_RSS', MAX_MEMORY_KB).to_i - # RSS in range (soft_limit_rss, hard_limit_rss) is allowed for GRACE_BALLOON_SECONDS - GRACE_BALLOON_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_GRACE_TIME', 15 * 60).to_i - # Check RSS every CHECK_INTERVAL_SECONDS, minimum 2 seconds - CHECK_INTERVAL_SECONDS = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max - # Give Sidekiq up to 30 seconds to allow existing jobs to finish after exceeding the limit - SHUTDOWN_TIMEOUT_SECONDS = ENV.fetch('SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT', 30).to_i - # Developer/admin should always set `memory_killer_max_memory_growth_kb` explicitly - # In case not set, default to 300M. This is for extra-safe. - DEFAULT_MAX_MEMORY_GROWTH_KB = 300_000 - - # Phases of memory killer - PHASE = { - running: 1, - above_soft_limit: 2, - stop_fetching_new_jobs: 3, - shutting_down: 4, - killing_sidekiq: 5 - }.freeze - - def initialize - super - - @enabled = true - @metrics = init_metrics - @sidekiq_daemon_monitor = Gitlab::SidekiqDaemon::Monitor.instance - end - - private - - def init_metrics - { - sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), - sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'), - sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'), - sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'), - sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached') - } - end - - def refresh_state(phase) - @phase = PHASE.fetch(phase) - @current_rss = get_rss_kb - @soft_limit_rss = get_soft_limit_rss_kb - @hard_limit_rss = get_hard_limit_rss_kb - @memory_total = get_memory_total_kb - - # track the current state as prometheus gauges - @metrics[:sidekiq_memory_killer_phase].set({}, @phase) - @metrics[:sidekiq_current_rss].set({}, @current_rss) - @metrics[:sidekiq_memory_killer_soft_limit_rss].set({}, @soft_limit_rss) - @metrics[:sidekiq_memory_killer_hard_limit_rss].set({}, @hard_limit_rss) - end - - def run_thread - Sidekiq.logger.info( - class: self.class.to_s, - action: 'start', - pid: pid, - message: 'Starting Gitlab::SidekiqDaemon::MemoryKiller Daemon' - ) - - while enabled? - begin - sleep(CHECK_INTERVAL_SECONDS) - restart_sidekiq unless rss_within_range? - rescue StandardError => e - log_exception(e, __method__) - rescue Exception => e # rubocop:disable Lint/RescueException - log_exception(e, __method__) - raise e - end - end - ensure - Sidekiq.logger.warn( - class: self.class.to_s, - action: 'stop', - pid: pid, - message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon' - ) - end - - def log_exception(exception, method) - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - message: "Exception from #{method}: #{exception.message}" - ) - end - - def stop_working - @enabled = false - end - - def enabled? - @enabled - end - - def restart_sidekiq - return if Feature.enabled?(:sidekiq_memory_killer_read_only_mode, type: :ops) - - # Tell Sidekiq to stop fetching new jobs - # We first SIGNAL and then wait given time - # We also monitor a number of running jobs and allow to restart early - refresh_state(:stop_fetching_new_jobs) - signal_and_wait(SHUTDOWN_TIMEOUT_SECONDS, 'SIGTSTP', 'stop fetching new jobs') - return unless enabled? - - # Tell sidekiq to restart itself - # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL - refresh_state(:shutting_down) - signal_and_wait(Sidekiq[:timeout] + 2, 'SIGTERM', 'gracefully shut down') - return unless enabled? - - # Ideally we should never reach this condition - # Wait for Sidekiq to shutdown gracefully, and kill it if it didn't - # Kill the whole pgroup, so we can be sure no children are left behind - refresh_state(:killing_sidekiq) - signal_pgroup('SIGKILL', 'die') - end - - def rss_within_range? - refresh_state(:running) - - deadline = Gitlab::Metrics::System.monotonic_time + GRACE_BALLOON_SECONDS.seconds - loop do - return true unless enabled? - - # RSS go above hard limit should trigger forcible shutdown right away - break if @current_rss > @hard_limit_rss - - # RSS go below the soft limit - return true if @current_rss < @soft_limit_rss - - # RSS did not go below the soft limit within deadline, restart - break if Gitlab::Metrics::System.monotonic_time > deadline - - sleep(CHECK_INTERVAL_SECONDS) - - refresh_state(:above_soft_limit) - - log_rss_out_of_range(false) - end - - # There are two chances to break from loop: - # - above hard limit, or - # - above soft limit after deadline - # When `above hard limit`, it immediately go to `stop_fetching_new_jobs` - # So ignore `above hard limit` and always set `above_soft_limit` here - refresh_state(:above_soft_limit) - log_rss_out_of_range - - false - end - - def log_rss_out_of_range(deadline_exceeded = true) - reason = out_of_range_description(@current_rss, - @hard_limit_rss, - @soft_limit_rss, - deadline_exceeded) - - running_jobs = fetch_running_jobs - - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - message: 'Sidekiq worker RSS out of range', - current_rss: @current_rss, - soft_limit_rss: @soft_limit_rss, - hard_limit_rss: @hard_limit_rss, - memory_total_kb: @memory_total, - reason: reason, - running_jobs: running_jobs) - - increment_worker_counters(running_jobs, deadline_exceeded) - end - - def increment_worker_counters(running_jobs, deadline_exceeded) - running_jobs.each do |job| - @metrics[:sidekiq_memory_killer_running_jobs].increment({ worker_class: job[:worker_class], deadline_exceeded: deadline_exceeded }) - end - end - - def fetch_running_jobs - @sidekiq_daemon_monitor.jobs.map do |jid, job| - { - jid: jid, - worker_class: job[:worker_class].name - } - end - end - - def out_of_range_description(rss, hard_limit, soft_limit, deadline_exceeded) - if rss > hard_limit - "current_rss(#{rss}) > hard_limit_rss(#{hard_limit})" - elsif deadline_exceeded - "current_rss(#{rss}) > soft_limit_rss(#{soft_limit}) longer than GRACE_BALLOON_SECONDS(#{GRACE_BALLOON_SECONDS})" - else - "current_rss(#{rss}) > soft_limit_rss(#{soft_limit})" - end - end - - def get_memory_total_kb - Gitlab::Metrics::System.memory_total / 1.kilobytes - end - - def get_rss_kb - Gitlab::Metrics::System.memory_usage_rss[:total] / 1.kilobytes - end - - def get_soft_limit_rss_kb - SOFT_LIMIT_RSS_KB + rss_increase_by_jobs - end - - def get_hard_limit_rss_kb - HARD_LIMIT_RSS_KB - end - - def signal_and_wait(time, signal, explanation) - Sidekiq.logger.warn( - class: self.class.to_s, - pid: pid, - signal: signal, - explanation: explanation, - wait_time: time, - message: "Sending signal and waiting" - ) - Process.kill(signal, pid) - - deadline = Gitlab::Metrics::System.monotonic_time + time - - # Sleep until thread killed or timeout reached - sleep(CHECK_INTERVAL_SECONDS) while enabled? && Gitlab::Metrics::System.monotonic_time < deadline - end - - def signal_pgroup(signal, explanation) - if Process.getpgrp == pid - pid_or_pgrp_str = 'PGRP' - pid_to_signal = 0 - else - pid_or_pgrp_str = 'PID' - pid_to_signal = pid - end - - Sidekiq.logger.warn( - class: self.class.to_s, - signal: signal, - pid: pid, - message: "sending Sidekiq worker #{pid_or_pgrp_str}-#{pid} #{signal} (#{explanation})" - ) - Process.kill(signal, pid_to_signal) - end - - def rss_increase_by_jobs - @sidekiq_daemon_monitor.jobs.sum do |_, job| - rss_increase_by_job(job) - end - end - - def rss_increase_by_job(job) - memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i - max_memory_growth_kb = get_job_options(job, 'memory_killer_max_memory_growth_kb', DEFAULT_MAX_MEMORY_GROWTH_KB).to_i - - return 0 if memory_growth_kb == 0 - - time_elapsed = [Gitlab::Metrics::System.monotonic_time - job[:started_at], 0].max - [memory_growth_kb * time_elapsed, max_memory_growth_kb].min - end - - def get_job_options(job, key, default) - job[:worker_class].sidekiq_options.fetch(key, default) - rescue StandardError - default - end - - def pid - Process.pid - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index b6e2209b475..3ed9c1743ed 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -147,7 +147,10 @@ module Gitlab end local cookie = cmsgpack.unpack(cookie_msgpack) cookie.deduplicated = "1" - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + local ttl = redis.call("ttl", KEYS[1]) + if ttl > 0 then + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl) + end LUA def should_reschedule? @@ -220,7 +223,12 @@ module Gitlab end def cookie_key - "#{idempotency_key}:cookie:v2" + # This duplicates `Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE` both here and in `#idempotency_key` + # This is because `Sidekiq.redis` used to add this prefix automatically through `redis-namespace` + # and we did not notice this in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/25447 + # Now we're keeping this as-is to avoid a key-migration when redis-namespace gets + # removed from Sidekiq: https://gitlab.com/groups/gitlab-com/gl-infra/-/epics/944 + "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{idempotency_key}:cookie:v2" end def get_cookie @@ -252,7 +260,7 @@ module Gitlab end def with_redis(&block) - Sidekiq.redis(&block) # rubocop:disable Cop/SidekiqRedisCall + Gitlab::Redis::Queues.with(&block) # rubocop:disable Cop/RedisQueueUsage, CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb index 0fc95534e2a..b065190f656 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -23,7 +23,7 @@ module Gitlab duplicate_job.set_deduplicated_flag!(expiry) Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log( - job, "dropped #{strategy_name}", duplicate_job.options) + job, strategy_name, duplicate_job.options) return false end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index e36f61be3b3..b3c3c94a0a3 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -17,6 +17,9 @@ module Gitlab SIDEKIQ_JOB_DURATION_BUCKETS = [10, 300].freeze SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze + # These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics + SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency].freeze + class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper @@ -47,17 +50,21 @@ module Gitlab return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) + possible_sli_labels = [] ::Gitlab::SidekiqConfig.current_worker_queue_mappings.each do |worker, queue| worker_class = worker.safe_constantize next unless worker_class base_labels = create_labels(worker_class, queue, {}) + possible_sli_labels << base_labels.slice(*SIDEKIQ_SLI_LABELS) %w[done fail].each do |status| metrics[:sidekiq_jobs_completion_seconds].get(base_labels.merge(job_status: status)) end end + + Gitlab::Metrics::SidekiqSlis.initialize_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) end end @@ -134,6 +141,12 @@ module Gitlab @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) end + + if ::Feature.enabled?(:sidekiq_execution_application_slis) + sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded + Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) + end end end diff --git a/lib/gitlab/slash_commands/global_slack_handler.rb b/lib/gitlab/slash_commands/global_slack_handler.rb new file mode 100644 index 00000000000..9bcc1f72b96 --- /dev/null +++ b/lib/gitlab/slash_commands/global_slack_handler.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + class GlobalSlackHandler + attr_reader :project_alias, :params + + def initialize(params) + @project_alias, command = parse_command_text(params) + @params = params.merge(text: command, original_command: params[:text]) + end + + def trigger + return false unless valid_token? + return Gitlab::SlashCommands::ApplicationHelp.new(nil, params).execute if help_command? + + unless slack_integration = find_slack_integration + error_message = 'GitLab error: project or alias not found' + return Gitlab::SlashCommands::Presenters::Error.new(error_message).message + end + + chat_user = ChatNames::FindUserService.new(params[:team_id], params[:user_id]).execute + integration = slack_integration.integration + + if chat_user&.user + Gitlab::SlashCommands::Command.new(integration.project, chat_user, params).execute + else + url = ChatNames::AuthorizeUserService.new(params).execute + Gitlab::SlashCommands::Presenters::Access.new(url).authorize + end + end + + private + + def valid_token? + ActiveSupport::SecurityUtils.secure_compare( + Gitlab::CurrentSettings.current_application_settings + .slack_app_verification_token, + params[:token] + ) + end + + def help_command? + params[:original_command] == 'help' + end + + # rubocop: disable CodeReuse/ActiveRecord + def find_slack_integration + if project_alias.nil? + SlackIntegration.find_by(team_id: params[:team_id]) + else + SlackIntegration.find_by(team_id: params[:team_id], alias: project_alias) + end + end + # rubocop: enable CodeReuse/ActiveRecord + + # Splits the command + # '/gitlab help' => [nil, 'help'] + # '/gitlab group/project issue new some title' => ['group/project', 'issue new some title'] + def parse_command_text(params) + if params[:text] == 'incident declare' + [nil, params[:text]] + else + fragments = params[:text].split(/\s/, 2) + fragments.size == 1 ? [nil, fragments.first] : fragments + end + end + end + end +end diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb index 722fcff151d..a43235bdeb6 100644 --- a/lib/gitlab/slash_commands/incident_management/incident_new.rb +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -5,11 +5,11 @@ module Gitlab module IncidentManagement class IncidentNew < IncidentCommand def self.help_message - 'incident declare' + 'incident declare *(Beta)*' end - def self.allowed?(project, user) - Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project) + def self.allowed?(_project, _user) + Feature.enabled?(:incident_declare_slash_command) end def self.match(text) diff --git a/lib/gitlab/slug/environment.rb b/lib/gitlab/slug/environment.rb index fd70def8e7c..2305fcd0061 100644 --- a/lib/gitlab/slug/environment.rb +++ b/lib/gitlab/slug/environment.rb @@ -21,7 +21,7 @@ module Gitlab slugified = name.to_s.downcase.gsub(/[^a-z0-9]/, '-') # Must start with a letter - slugified = 'env-' + slugified unless slugified.match?(/^[a-z]/) + slugified = +"env-#{slugified}" unless slugified.match?(/^[a-z]/) # Repeated dashes are invalid (OpenShift limitation) slugified.squeeze!('-') diff --git a/lib/gitlab/source.rb b/lib/gitlab/source.rb new file mode 100644 index 00000000000..0e9fb39156d --- /dev/null +++ b/lib/gitlab/source.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + class Source # rubocop:disable Gitlab/NamespacedClass + class << self + def ref + return Gitlab.revision if Gitlab.pre_release? + + "v#{Gitlab::VERSION}" + end + + def release_url + path = if Gitlab.pre_release? + url_helpers.namespace_project_commits_path(group, project, ref) + else + url_helpers.namespace_project_tag_path(group, project, ref) + end + + Gitlab::Utils.append_path(host_url, path) + end + + private + + def host_url + Gitlab::Saas.com_url + end + + def group + 'gitlab-org' + end + + def project + 'gitlab-foss' + end + + def url_helpers + Rails.application.routes.url_helpers + end + end + end +end + +Gitlab::Source.prepend_mod diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 0b9f3baa4de..222dd54b7b4 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -3,19 +3,13 @@ require 'spamcheck' module Gitlab module Spamcheck + Error = Class.new(StandardError) + class Client include ::Spam::SpamConstants DEFAULT_TIMEOUT_SECS = 2 - VERDICT_MAPPING = { - ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW, - ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW, - ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW, - ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER, - ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP - }.freeze - ACTION_MAPPING = { create: ::Spamcheck::Action::CREATE, update: ::Spamcheck::Action::UPDATE @@ -40,8 +34,9 @@ module Gitlab pb, grpc_method = build_protobuf(**protobuf_args) response = grpc_method.call(pb, metadata: metadata) - verdict = convert_verdict_to_gitlab_constant(response.verdict) - [verdict, response.extra_attributes.to_h, response.error] + raise Error, response.error unless response.error.blank? + + Result.new(response) end private @@ -53,19 +48,16 @@ module Gitlab when Snippet [::Spamcheck::Snippet, grpc_client.method(:check_for_spam_snippet)] else - raise ArgumentError, "Not a spammable type: #{spammable.class.name}" + [::Spamcheck::Generic, grpc_client.method(:check_for_spam_generic)] end end - def convert_verdict_to_gitlab_constant(verdict) - VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict) - end - def build_protobuf(spammable:, user:, context:, extra_features:) protobuf_class, grpc_method = get_spammable_mappings(spammable) pb = protobuf_class.new(**extra_features) - pb.title = spammable.spam_title || '' - pb.description = spammable.spam_description || '' + pb.title = spammable.spam_title || '' if pb.respond_to?(:title) + pb.description = spammable.spam_description || '' if pb.respond_to?(:description) + pb.text = spammable.spammable_text || '' if pb.respond_to?(:text) pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) @@ -82,8 +74,10 @@ module Gitlab def build_user_protobuf(user) user_pb = ::Spamcheck::User.new user_pb.username = user.username + user_pb.id = user.id user_pb.org = user.organization || '' user_pb.created_at = convert_to_pb_timestamp(user.created_at) + user_pb.abuse_metadata = Google::Protobuf::Map.new(:string, :float, user.abuse_metadata) user_pb.emails << build_email(user.email, user.confirmed?) diff --git a/lib/gitlab/spamcheck/result.rb b/lib/gitlab/spamcheck/result.rb new file mode 100644 index 00000000000..7cf6b020bf5 --- /dev/null +++ b/lib/gitlab/spamcheck/result.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Gitlab + module Spamcheck + class Result + include ::Spam::SpamConstants + attr_reader :response + + VERDICT_MAPPING = { + ::Spamcheck::SpamVerdict::Verdict::ALLOW => ALLOW, + ::Spamcheck::SpamVerdict::Verdict::CONDITIONAL_ALLOW => CONDITIONAL_ALLOW, + ::Spamcheck::SpamVerdict::Verdict::DISALLOW => DISALLOW, + ::Spamcheck::SpamVerdict::Verdict::BLOCK => BLOCK_USER, + ::Spamcheck::SpamVerdict::Verdict::NOOP => NOOP + }.freeze + + def initialize(response) + @response = response + end + + def score + response.score + end + + def verdict + VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(response.verdict), ALLOW) + end + + def evaluated? + response.evaluated + end + end + end +end diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 7494f0584d0..1d9ecb624b2 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -2,22 +2,6 @@ module Gitlab module SubscriptionPortal - def self.default_subscriptions_url - if ::Gitlab.dev_or_test_env? - 'https://customers.staging.gitlab.com' - else - 'https://customers.gitlab.com' - end - end - - def self.subscriptions_url - ENV.fetch('CUSTOMER_PORTAL_URL', default_subscriptions_url) - end - - def self.payment_form_url - "#{self.subscriptions_url}/payment_forms/cc_validation" - end - def self.payment_validation_form_id "payment_method_validation" end @@ -26,54 +10,6 @@ module Gitlab "cc_registration_validation" end - def self.registration_validation_form_url - "#{self.subscriptions_url}/payment_forms/cc_registration_validation" - end - - def self.subscriptions_comparison_url - 'https://about.gitlab.com/pricing/gitlab-com/feature-comparison' - end - - def self.subscriptions_graphql_url - "#{self.subscriptions_url}/graphql" - end - - def self.subscriptions_more_minutes_url - "#{self.subscriptions_url}/buy_pipeline_minutes" - end - - def self.subscriptions_more_storage_url - "#{self.subscriptions_url}/buy_storage" - end - - def self.subscriptions_manage_url - "#{self.subscriptions_url}/subscriptions" - end - - def self.subscriptions_gitlab_plans_url - "#{self.subscriptions_url}/gitlab_plans" - end - - def self.subscriptions_instance_review_url - "#{self.subscriptions_url}/instance_review" - end - - def self.add_extra_seats_url(group_id) - "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/extra_seats" - end - - def self.upgrade_subscription_url(group_id, plan_id) - "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/upgrade/#{plan_id}" - end - - def self.renew_subscription_url(group_id) - "#{self.subscriptions_url}/gitlab/namespaces/#{group_id}/renew" - end - - def self.edit_account_url - "#{self.subscriptions_url}/customers/edit" - end - def self.subscription_portal_admin_email ENV.fetch('SUBSCRIPTION_PORTAL_ADMIN_EMAIL', 'gl_com_api@gitlab.com') end @@ -89,9 +25,8 @@ module Gitlab end Gitlab::SubscriptionPortal.prepend_mod -Gitlab::SubscriptionPortal::SUBSCRIPTIONS_URL = Gitlab::SubscriptionPortal.subscriptions_url.freeze -Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.payment_form_url.freeze Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze -Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.registration_validation_form_id.freeze +Gitlab::SubscriptionPortal::SUBSCRIPTION_PORTAL_ADMIN_EMAIL = Gitlab::SubscriptionPortal.subscription_portal_admin_email.freeze +Gitlab::SubscriptionPortal::SUBSCRIPTION_PORTAL_ADMIN_TOKEN = Gitlab::SubscriptionPortal.subscription_portal_admin_token.freeze diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index 9dba8c99b99..b9800a4db73 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require_dependency 'gitlab/utils/strong_memoize' +require_relative 'utils/strong_memoize' # rubocop:disable Rails/Output module Gitlab diff --git a/lib/gitlab/timeless.rb b/lib/gitlab/timeless.rb index ed0b7b4ed87..55cd9d7e6fb 100644 --- a/lib/gitlab/timeless.rb +++ b/lib/gitlab/timeless.rb @@ -2,17 +2,11 @@ module Gitlab module Timeless - def self.timeless(model, &block) + def self.timeless(model) original_record_timestamps = model.record_timestamps model.record_timestamps = false - # negative arity means arguments are optional - if block.arity == 1 || block.arity < 0 - yield(model) - else - yield - end - + yield model ensure model.record_timestamps = original_record_timestamps end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 45f836f10d3..52aee4d2d45 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -8,13 +8,35 @@ module Gitlab end def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists - contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] + action = action.to_s + + project_id = project.is_a?(Integer) ? project : project&.id + + contexts = [ + Tracking::StandardContext.new( + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name, + project_id: project_id, + user_id: user&.id, + **extra).to_context, *context + ] + track_struct_event(tracker, category, action, label: label, property: property, value: value, contexts: contexts) + end + + def database_event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists action = action.to_s + destination = Gitlab::Tracking::Destinations::DatabaseEventsSnowplow.new + contexts = [ + Tracking::StandardContext.new( + namespace_id: namespace&.id, + plan_name: namespace&.actual_plan_name, + project_id: project&.id, + user_id: user&.id, + **extra).to_context, *context + ] - tracker.event(category, action, label: label, property: property, value: value, context: contexts) - rescue StandardError => error - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) + track_struct_event(destination, category, action, label: label, property: property, value: value, contexts: contexts) end def definition(basename, category: nil, action: nil, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists @@ -42,12 +64,19 @@ module Gitlab def snowplow_micro_enabled? Rails.env.development? && Gitlab.config.snowplow_micro.enabled - rescue Settingslogic::MissingSetting + rescue GitlabSettings::MissingSetting false end private + def track_struct_event(destination, category, action, label:, property:, value:, contexts:) # rubocop:disable Metrics/ParameterLists + destination + .event(category, action, label: label, property: property, value: value, context: contexts) + rescue StandardError => error + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) + end + def tracker @tracker ||= if snowplow_micro_enabled? Gitlab::Tracking::Destinations::SnowplowMicro.new diff --git a/lib/gitlab/tracking/destinations/database_events_snowplow.rb b/lib/gitlab/tracking/destinations/database_events_snowplow.rb new file mode 100644 index 00000000000..e3512bc4916 --- /dev/null +++ b/lib/gitlab/tracking/destinations/database_events_snowplow.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Destinations + class DatabaseEventsSnowplow < Snowplow + extend ::Gitlab::Utils::Override + + HOSTNAME = 'db-snowplow.trx.gitlab.net' + + override :enabled? + # database events are only collected for SaaS instance + def enabled? + ::Gitlab.dev_or_test_env? || ::Gitlab.com? + end + + override :hostname + def hostname + return HOSTNAME if ::Gitlab.com? + + 'localhost:9091' + end + + private + + override :increment_failed_events_emissions + def increment_failed_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_db_events_snowplow_failed_events_total, + 'Number of failed Snowplow events emissions' + ).increment({}, value.to_i) + end + + override :increment_successful_events_emissions + def increment_successful_events_emissions(value) + Gitlab::Metrics.counter( + :gitlab_db_events_snowplow_successful_events_total, + 'Number of successful Snowplow events emissions' + ).increment({}, value.to_i) + end + + override :increment_total_events_counter + def increment_total_events_counter + Gitlab::Metrics.counter( + :gitlab_db_events_snowplow_events_total, + 'Number of Snowplow events' + ).increment + end + end + end + end +end diff --git a/lib/gitlab/tracking/destinations/snowplow_micro.rb b/lib/gitlab/tracking/destinations/snowplow_micro.rb index 09480f26106..e15c03b6808 100644 --- a/lib/gitlab/tracking/destinations/snowplow_micro.rb +++ b/lib/gitlab/tracking/destinations/snowplow_micro.rb @@ -53,7 +53,7 @@ module Gitlab url = Gitlab.config.snowplow_micro.address scheme = Gitlab.config.gitlab.https ? 'https' : 'http' "#{scheme}://#{url}" - rescue Settingslogic::MissingSetting + rescue GitlabSettings::MissingSetting DEFAULT_URI end end diff --git a/lib/gitlab/tracking/standard_context.rb b/lib/gitlab/tracking/standard_context.rb index 50467de44b8..62c45368410 100644 --- a/lib/gitlab/tracking/standard_context.rb +++ b/lib/gitlab/tracking/standard_context.rb @@ -6,15 +6,16 @@ module Gitlab GITLAB_STANDARD_SCHEMA_URL = 'iglu:com.gitlab/gitlab_standard/jsonschema/1-0-8' GITLAB_RAILS_SOURCE = 'gitlab-rails' - def initialize(namespace: nil, project: nil, user: nil, **extra) - check_argument_type(:namespace, namespace, [Namespace]) - check_argument_type(:project, project, [Project, Integer]) - check_argument_type(:user, user, [User, DeployToken]) - - @namespace = namespace - @plan = namespace&.actual_plan_name - @project = project - @user = user + def initialize(namespace_id: nil, plan_name: nil, project_id: nil, user_id: nil, **extra) + check_argument_type(:namespace_id, namespace_id, [Integer]) + check_argument_type(:plan_name, plan_name, [String]) + check_argument_type(:project_id, project_id, [Integer]) + check_argument_type(:user_id, user_id, [Integer]) + + @namespace_id = namespace_id + @plan_name = plan_name + @project_id = project_id + @user_id = user_id @extra = extra end @@ -40,25 +41,21 @@ module Gitlab private - attr_accessor :namespace, :project, :extra, :plan, :user + attr_accessor :namespace_id, :project_id, :extra, :plan_name, :user_id def to_h { environment: environment, source: source, - plan: plan, + plan: plan_name, extra: extra, - user_id: user&.id, - namespace_id: namespace&.id, + user_id: user_id, + namespace_id: namespace_id, project_id: project_id, context_generated_at: Time.current } end - def project_id - project.is_a?(Integer) ? project : project&.id - end - def check_argument_type(argument_name, argument_value, allowed_classes) return if argument_value.nil? || allowed_classes.any? { |allowed_class| argument_value.is_a?(allowed_class) } diff --git a/lib/gitlab/untrusted_regexp.rb b/lib/gitlab/untrusted_regexp.rb index 7c7bda3a8f9..fe3377dae68 100644 --- a/lib/gitlab/untrusted_regexp.rb +++ b/lib/gitlab/untrusted_regexp.rb @@ -10,6 +10,9 @@ module Gitlab # Not all regular expression features are available in untrusted regexes, and # there is a strict limit on total execution time. See the RE2 documentation # at https://github.com/google/re2/wiki/Syntax for more details. + # + # This class doesn't change any instance variables, which allows it to be frozen + # and setup in constants. class UntrustedRegexp require_dependency 're2' @@ -21,6 +24,7 @@ module Gitlab end @regexp = RE2::Regexp.new(pattern, log_errors: false) + @scan_regexp = initialize_scan_regexp raise RegexpError, regexp.error unless regexp.ok? end @@ -29,6 +33,27 @@ module Gitlab RE2.GlobalReplace(text, regexp, rewrite) end + # There is no built-in replace with block support (like `gsub`). We can accomplish + # the same thing by parsing and rebuilding the string with the substitutions. + def replace_gsub(text) + new_text = +'' + remainder = text + + matched = match(remainder) + + until matched.nil? || matched.to_a.compact.empty? + partitioned = remainder.partition(matched.to_s) + new_text << partitioned.first + remainder = partitioned.last + + new_text << yield(matched) + + matched = match(remainder) + end + + new_text << remainder + end + def scan(text) matches = scan_regexp.scan(text).to_a matches.map!(&:first) if regexp.number_of_capturing_groups == 0 @@ -87,17 +112,16 @@ module Gitlab private - attr_reader :regexp + attr_reader :regexp, :scan_regexp # RE2 scan operates differently to Ruby scan when there are no capture # groups, so work around it - def scan_regexp - @scan_regexp ||= - if regexp.number_of_capturing_groups == 0 - RE2::Regexp.new('(' + regexp.source + ')') - else - regexp - end + def initialize_scan_regexp + if regexp.number_of_capturing_groups == 0 + RE2::Regexp.new('(' + regexp.source + ')') + else + regexp + end end end end diff --git a/lib/gitlab/url_blocker.rb b/lib/gitlab/url_blocker.rb index 00e609511f2..ba50a42cd37 100644 --- a/lib/gitlab/url_blocker.rb +++ b/lib/gitlab/url_blocker.rb @@ -7,20 +7,40 @@ module Gitlab class UrlBlocker BlockedUrlError = Class.new(StandardError) + DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT = proc { deny_all_requests_except_allowed_app_setting }.freeze + + # Result stores the validation result: + # uri - The original URI requested + # hostname - The hostname that should be used to connect. For DNS + # rebinding protection, this will be the resolved IP address of + # the hostname. + # use_proxy - + # If true, this means that the proxy server specified in the + # http_proxy/https_proxy environment variables should be used. + # + # If false, this either means that no proxy server was specified + # or that the hostname in the URL is exempt via the no_proxy + # environment variable. This allows the caller to disable usage + # of a proxy since the IP address may be used to + # connect. Otherwise, Net::HTTP may erroneously compare the IP + # address against the no_proxy list. + Result = Struct.new(:uri, :hostname, :use_proxy) + class << self # Validates the given url according to the constraints specified by arguments. # - # ports - Raises error if the given URL port does is not between given ports. + # ports - Raises error if the given URL port is not between given ports. # allow_localhost - Raises error if URL resolves to a localhost IP address and argument is false. # allow_local_network - Raises error if URL resolves to a link-local address and argument is false. # allow_object_storage - Avoid raising an error if URL resolves to an object storage endpoint and argument is true. # ascii_only - Raises error if URL has unicode characters and argument is true. # enforce_user - Raises error if URL user doesn't start with alphanumeric characters and argument is true. # enforce_sanitization - Raises error if URL includes any HTML/CSS/JS tags and argument is true. + # deny_all_requests_except_allowed - Raises error if URL is not in the allow list and argument is true. Can be Boolean or Proc. Defaults to instance app setting. # - # Returns an array with [<uri>, <original-hostname>]. + # Returns a Result object. # rubocop:disable Metrics/ParameterLists - def validate!( + def validate_url_with_proxy!( url, schemes:, ports: [], @@ -30,10 +50,11 @@ module Gitlab ascii_only: false, enforce_user: false, enforce_sanitization: false, + deny_all_requests_except_allowed: DENY_ALL_REQUESTS_EXCEPT_ALLOWED_DEFAULT, dns_rebind_protection: true) # rubocop:enable Metrics/ParameterLists - return [nil, nil] if url.nil? + return Result.new(nil, nil, true) if url.nil? raise ArgumentError, 'The schemes is a required argument' if schemes.blank? @@ -49,21 +70,35 @@ module Gitlab ascii_only: ascii_only ) - address_info = get_address_info(uri, dns_rebind_protection) - return [uri, nil] unless address_info + begin + address_info = get_address_info(uri) + rescue SocketError + proxy_in_use = uri_under_proxy_setting?(uri, nil) + + return Result.new(uri, nil, proxy_in_use) unless enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) + + raise BlockedUrlError, 'Host cannot be resolved or invalid' + end ip_address = ip_address(address_info) - return [uri, nil] if domain_allowed?(uri) + proxy_in_use = uri_under_proxy_setting?(uri, ip_address) + + # Ignore DNS rebind protection when a proxy is being used, as DNS + # rebinding is expected behavior. + dns_rebind_protection &&= !proxy_in_use + return Result.new(uri, nil, proxy_in_use) if domain_in_allow_list?(uri) - protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection) + protected_uri_with_hostname = enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use) - return protected_uri_with_hostname if ip_allowed?(ip_address, port: get_port(uri)) + return protected_uri_with_hostname if ip_in_allow_list?(ip_address, port: get_port(uri)) # Allow url from the GitLab instance itself but only for the configured hostname and ports return protected_uri_with_hostname if internal?(uri) return protected_uri_with_hostname if allow_object_storage && object_storage_endpoint?(uri) + validate_deny_all_requests_except_allowed!(deny_all_requests_except_allowed) + validate_local_request( address_info: address_info, allow_localhost: allow_localhost, @@ -81,6 +116,13 @@ module Gitlab true end + # For backwards compatibility, Returns an array with [<uri>, <original-hostname>]. + # Issue for refactoring: https://gitlab.com/gitlab-org/gitlab/-/issues/410890 + def validate!(...) + result = validate_url_with_proxy!(...) + [result.uri, result.hostname] + end + private # Returns the given URI with IP address as hostname and the original hostname respectively @@ -91,12 +133,12 @@ module Gitlab # # The original hostname is used to validate the SSL, given in that scenario # we'll be making the request to the IP address, instead of using the hostname. - def enforce_uri_hostname(ip_address, uri, dns_rebind_protection) - return [uri, nil] unless dns_rebind_protection && ip_address && ip_address != uri.hostname + def enforce_uri_hostname(ip_address, uri, dns_rebind_protection, proxy_in_use) + return Result.new(uri, nil, proxy_in_use) unless dns_rebind_protection && ip_address && ip_address != uri.hostname new_uri = uri.dup new_uri.hostname = ip_address - [new_uri, uri.hostname] + Result.new(new_uri, uri.hostname, proxy_in_use) end def ip_address(address_info) @@ -115,29 +157,56 @@ module Gitlab validate_unicode_restriction(uri) if ascii_only end - def get_address_info(uri, dns_rebind_protection) + def uri_under_proxy_setting?(uri, ip_address) + return false unless Gitlab.http_proxy_env? + # `no_proxy|NO_PROXY` specifies addresses for which the proxy is not + # used. If it's empty, there are no exceptions and this URI + # will be under proxy settings. + return true if no_proxy_env.blank? + + # `no_proxy|NO_PROXY` is being used. We must check whether it + # applies to this specific URI. + ::URI::Generic.use_proxy?(uri.hostname, ip_address, get_port(uri), no_proxy_env) + end + + # Returns addrinfo object for the URI. + # + # @param uri [Addressable::URI] + # + # @raise [Gitlab::UrlBlocker::BlockedUrlError, ArgumentError] - BlockedUrlError raised if host is too long. + # + # @return [Array<Addrinfo>] + def get_address_info(uri) Addrinfo.getaddrinfo(uri.hostname, get_port(uri), nil, :STREAM).map do |addr| addr.ipv6_v4mapped? ? addr.ipv6_to_ipv4 : addr end - rescue SocketError - # If the dns rebinding protection is not enabled or the domain - # is allowed we avoid the dns rebinding checks - return if domain_allowed?(uri) || !dns_rebind_protection + rescue ArgumentError => error + # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. + raise unless error.message.include?('hostname too long') + + raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + end + + def enforce_address_info_retrievable?(uri, dns_rebind_protection, deny_all_requests_except_allowed) + # Do not enforce if URI is in the allow list + return false if domain_in_allow_list?(uri) + + # Enforce if the instance should block requests + return true if deny_all_requests_except_allowed?(deny_all_requests_except_allowed) + + # Do not enforce if DNS rebinding protection is disabled + return false unless dns_rebind_protection + + # Do not enforce if proxy is used + return false if Gitlab.http_proxy_env? # In the test suite we use a lot of mocked urls that are either invalid or # don't exist. In order to avoid modifying a ton of tests and factories # we allow invalid urls unless the environment variable RSPEC_ALLOW_INVALID_URLS # is not true - return if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' - - # If the addr can't be resolved or the url is invalid (i.e http://1.1.1.1.1) - # we block the url - raise BlockedUrlError, "Host cannot be resolved or invalid" - rescue ArgumentError => error - # Addrinfo.getaddrinfo errors if the domain exceeds 1024 characters. - raise unless error.message.include?('hostname too long') + return false if Rails.env.test? && ENV['RSPEC_ALLOW_INVALID_URLS'] == 'true' - raise BlockedUrlError, "Host is too long (maximum is 1024 characters)" + true end def validate_local_request( @@ -260,6 +329,15 @@ module Gitlab raise BlockedUrlError, "Requests to the link local network are not allowed" end + # Raises a BlockedUrlError if the instance is configured to deny all requests. + # + # This should only be called after allow list checks have been made. + def validate_deny_all_requests_except_allowed!(should_deny) + return unless deny_all_requests_except_allowed?(should_deny) + + raise BlockedUrlError, "Requests to hosts and IP addresses not on the Allow List are denied" + end + # Raises a BlockedUrlError if any IP in `addrs_info` is the limited # broadcast address. # https://datatracker.ietf.org/doc/html/rfc919#section-7 @@ -293,8 +371,7 @@ module Gitlab next unless section_setting && section_setting['enabled'] - # Use #to_h to avoid Settingslogic bug: https://gitlab.com/gitlab-org/gitlab/-/issues/286873 - object_store_setting = section_setting['object_store']&.to_h + object_store_setting = section_setting['object_store'] next unless object_store_setting && object_store_setting['enabled'] @@ -302,6 +379,15 @@ module Gitlab end.compact.uniq end + def deny_all_requests_except_allowed?(should_deny) + should_deny.is_a?(Proc) ? should_deny.call : should_deny + end + + def deny_all_requests_except_allowed_app_setting + Gitlab::CurrentSettings.current_application_settings? && + Gitlab::CurrentSettings.deny_all_requests_except_allowed? + end + def object_storage_endpoint?(uri) enabled_object_storage_endpoints.any? do |endpoint| endpoint_uri = URI(endpoint) @@ -312,17 +398,21 @@ module Gitlab end end - def domain_allowed?(uri) + def domain_in_allow_list?(uri) Gitlab::UrlBlockers::UrlAllowlist.domain_allowed?(uri.normalized_host, port: get_port(uri)) end - def ip_allowed?(ip_address, port: nil) + def ip_in_allow_list?(ip_address, port: nil) Gitlab::UrlBlockers::UrlAllowlist.ip_allowed?(ip_address, port: port) end def config Gitlab.config end + + def no_proxy_env + ENV['no_proxy'] || ENV['NO_PROXY'] + end end end end diff --git a/lib/gitlab/url_blockers/ip_allowlist_entry.rb b/lib/gitlab/url_blockers/ip_allowlist_entry.rb index b293afe166c..ff4eb86ec41 100644 --- a/lib/gitlab/url_blockers/ip_allowlist_entry.rb +++ b/lib/gitlab/url_blockers/ip_allowlist_entry.rb @@ -12,11 +12,32 @@ module Gitlab end def match?(requested_ip, requested_port = nil) - return false unless ip.include?(requested_ip) + requested_ip = IPAddr.new(requested_ip) if requested_ip.is_a?(String) + + return false unless ip_include?(requested_ip) return true if port.nil? port == requested_port end + + private + + # Prior to ipaddr v1.2.3, if the allow list were the IPv4 to IPv6 + # mapped address ::ffff:169.254.168.100 and the requested IP were + # 169.254.168.100 or ::ffff:169.254.168.100, the IP would be + # considered in the allow list. However, with + # https://github.com/ruby/ipaddr/pull/31, IPAddr#include? will + # only match if the IP versions are the same. This method + # preserves backwards compatibility if the versions differ by + # checking inclusion by coercing an IPv4 address to its IPv6 + # mapped address. + def ip_include?(requested_ip) + return true if ip.include?(requested_ip) + return ip.include?(requested_ip.ipv4_mapped) if requested_ip.ipv4? && ip.ipv6? + return ip.ipv4_mapped.include?(requested_ip) if requested_ip.ipv6? && ip.ipv4? + + false + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index b68e1ace658..a0a58534661 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -7,11 +7,6 @@ module Gitlab class Aggregate include Gitlab::Usage::TimeFrame - # TODO: define this missing event https://gitlab.com/gitlab-org/gitlab/-/issues/385080 - EVENTS_NOT_DEFINED_YET = %w[ - i_code_review_merge_request_widget_license_compliance_warning - ].freeze - def initialize(recorded_at) @recorded_at = recorded_at end @@ -84,7 +79,7 @@ module Gitlab return events if source != ::Gitlab::Usage::Metrics::Aggregates::REDIS_SOURCE events.select do |event| - ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) || EVENTS_NOT_DEFINED_YET.include?(event) + ::Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(event) end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index 642b67a3b02..ca122ccf6f3 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -23,6 +23,7 @@ module Gitlab scope = super scope = scope.where(source_type: source_type) if source_type.present? scope = scope.where(status: status) if status.present? + scope = scope.where(has_failures: failures) if failures.present? scope end @@ -34,6 +35,10 @@ module Gitlab options[:status] end + def failures + options[:has_failures].to_s + end + def allowed_source_types BulkImports::Entity.source_types.keys.map(&:to_s) end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb new file mode 100644 index 00000000000..fbf4e0f904b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersGroupTypeActiveMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner.group_type.active } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb new file mode 100644 index 00000000000..acb6de53d14 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_group_type_active_online_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersGroupTypeActiveOnlineMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner.group_type.active.online } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb new file mode 100644 index 00000000000..d9a785679d7 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersInstanceTypeActiveMetric < DatabaseMetric + operation :count + + relation do + ::Ci::Runner.instance_type.active + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb new file mode 100644 index 00000000000..05a9c47c016 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_instance_type_active_online_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersInstanceTypeActiveOnlineMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner.instance_type.active.online } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb new file mode 100644 index 00000000000..8be4955e28d --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb new file mode 100644 index 00000000000..e713e85b270 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersProjectTypeActiveMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner.project_type.active } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb new file mode 100644 index 00000000000..91e7c6063b8 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_ci_runners_project_type_active_online_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountCiRunnersProjectTypeActiveOnlineMetric < DatabaseMetric + operation :count + + relation { ::Ci::Runner.project_type.active.online } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_mode.rb b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb new file mode 100644 index 00000000000..1b97ef4a1d2 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/database_mode.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DatabaseMode < GenericMetric + value do + Gitlab::Database.database_mode + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb b/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb new file mode 100644 index 00000000000..83153242703 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/edition_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class EditionMetric < GenericMetric + value do + if Gitlab.ee? + ::License.current&.edition || 'EE Free' + else + 'CE' + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb new file mode 100644 index 00000000000..b7ca5fadd5b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitlab_dedicated_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitlabDedicatedMetric < GenericMetric + value do + Gitlab::CurrentSettings.gitlab_dedicated_instance + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb index ab9c6f87023..be3b3b3d682 100644 --- a/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/incoming_email_encrypted_secrets_enabled_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class IncomingEmailEncryptedSecretsEnabledMetric < GenericMetric value do - Gitlab::IncomingEmail.encrypted_secrets.active? + Gitlab::Email::IncomingEmail.encrypted_secrets.active? end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb new file mode 100644 index 00000000000..409027925d1 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class IndexInconsistenciesMetric < GenericMetric + value do + runner = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database, validators: validators) + + inconsistencies = runner.execute + + inconsistencies.map do |inconsistency| + { + object_name: inconsistency.object_name, + inconsistency_type: inconsistency.type + } + end + end + + class << self + private + + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Database::SchemaValidation::Database.new(database_model.connection) + end + + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) + end + + def validators + [ + Gitlab::Database::SchemaValidation::Validators::MissingIndexes, + Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, + Gitlab::Database::SchemaValidation::Validators::ExtraIndexes + ] + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb new file mode 100644 index 00000000000..e8ae4b4f906 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_approximation_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InstallationCreationDateApproximationMetric < GenericMetric + value do + [User.first, ApplicationSetting.first].compact.pluck(:created_at).compact.min + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb new file mode 100644 index 00000000000..c2ca62f9eba --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/installation_creation_date_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InstallationCreationDateMetric < GenericMetric + value do + User.where(id: 1).pick(:created_at) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb b/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb new file mode 100644 index 00000000000..7147fc2e624 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/installation_type_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InstallationTypeMetric < GenericMetric + value do + if Rails.env.production? + Gitlab::INSTALLATION_TYPE + else + "gitlab-development-kit" + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb index 4332043de8a..5e38339801b 100644 --- a/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/service_desk_email_encrypted_secrets_enabled_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class ServiceDeskEmailEncryptedSecretsEnabledMetric < GenericMetric value do - Gitlab::ServiceDeskEmail.encrypted_secrets.active? + Gitlab::Email::ServiceDeskEmail.encrypted_secrets.active? end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/version_metric.rb b/lib/gitlab/usage/metrics/instrumentations/version_metric.rb new file mode 100644 index 00000000000..cc26268067f --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/version_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class VersionMetric < GenericMetric + value do + Gitlab::VERSION + end + end + end + end + end +end diff --git a/lib/gitlab/usage/service_ping_report.rb b/lib/gitlab/usage/service_ping_report.rb index 1eda72ba570..3bc941260d6 100644 --- a/lib/gitlab/usage/service_ping_report.rb +++ b/lib/gitlab/usage/service_ping_report.rb @@ -9,7 +9,9 @@ module Gitlab def for(output:, cached: false) case output.to_sym when :all_metrics_values - with_instrumentation_classes(all_metrics_values(cached), :with_value) + Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do + with_instrumentation_classes(Gitlab::UsageData.data, :with_value) + end when :metrics_queries with_instrumentation_classes(metrics_queries, :with_instrumentation) when :non_sql_metrics_values @@ -27,12 +29,6 @@ module Gitlab old_payload.with_indifferent_access.deep_merge(instrumented_payload) end - def all_metrics_values(cached) - Rails.cache.fetch(CACHE_KEY, force: !cached, expires_in: 2.weeks) do - Gitlab::UsageData.data - end - end - def metrics_queries Gitlab::UsageDataQueries.data end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 53794854bd0..846bb934a3d 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -47,15 +47,7 @@ module Gitlab end def license_usage_data - { - recorded_at: recorded_at, - uuid: add_metric('UuidMetric'), - hostname: add_metric('HostnameMetric'), - version: alt_usage_data { Gitlab::VERSION }, - installation_type: alt_usage_data { installation_type }, - active_user_count: add_metric('ActiveUserCountMetric'), - edition: 'CE' - } + { recorded_at: recorded_at } end def recorded_at @@ -145,7 +137,6 @@ module Gitlab merge_requests: count(MergeRequest), notes: count(Note) }.merge( - runners_usage, integrations_usage, user_preferences_usage, service_desk_counts @@ -156,18 +147,6 @@ module Gitlab end # rubocop: enable Metrics/AbcSize - def runners_usage - { - ci_runners: count(::Ci::Runner), - ci_runners_instance_type_active: count(::Ci::Runner.instance_type.active), - ci_runners_group_type_active: count(::Ci::Runner.group_type.active), - ci_runners_project_type_active: count(::Ci::Runner.project_type.active), - ci_runners_instance_type_active_online: count(::Ci::Runner.instance_type.active.online), - ci_runners_group_type_active_online: count(::Ci::Runner.group_type.active.online), - ci_runners_project_type_active_online: count(::Ci::Runner.project_type.active.online) - } - end - def system_usage_data_monthly { counts_monthly: { @@ -240,7 +219,7 @@ module Gitlab omniauth_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth.omniauth_enabled? }, prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, - reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::IncomingEmail.enabled? }, + reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? }, web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, @@ -333,24 +312,10 @@ module Gitlab end def jira_usage - # Jira Cloud does not support custom domains as per https://jira.atlassian.com/browse/CLOUD-6999 - # so we can just check for subdomains of atlassian.net - jira_integration_data_hash = jira_integration_data - if jira_integration_data_hash.nil? - return { projects_jira_server_active: FALLBACK, projects_jira_cloud_active: FALLBACK } - end - - results = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0, + { projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled), projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) } - - results[:projects_jira_server_active] = jira_integration_data_hash[:projects_jira_server_active] - results[:projects_jira_cloud_active] = jira_integration_data_hash[:projects_jira_cloud_active] - - results end # rubocop: enable CodeReuse/ActiveRecord @@ -384,33 +349,13 @@ module Gitlab } end - def merge_requests_users(time_period) - counter = Gitlab::UsageDataCounters::TrackUniqueEvents - - redis_usage_data do - counter.count_unique_events( - event_action: Gitlab::UsageDataCounters::TrackUniqueEvents::MERGE_REQUEST_ACTION, - date_from: time_period[:created_at].first, - date_to: time_period[:created_at].last - ) - end - end - - def installation_type - if Rails.env.production? - Gitlab::INSTALLATION_TYPE - else - "gitlab-development-kit" - end - end - def operating_system ohai_data = Ohai::System.new.tap do |oh| oh.all_plugins(['platform']) end.data platform = ohai_data['platform'] - platform = 'raspbian' if ohai_data['platform'] == 'debian' && /armv/.match?(ohai_data['kernel']['machine']) + platform = 'raspbian' if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') "#{platform}-#{ohai_data['platform_version']}" end @@ -463,12 +408,7 @@ module Gitlab projects_without_disable_overriding_approvers_per_merge_request: count(::Project.where(time_period.merge(disable_overriding_approvers_per_merge_request: [false, nil]))), remote_mirrors: distinct_count(::Project.with_remote_mirrors.where(time_period), :creator_id), snippets: distinct_count(::Snippet.where(time_period), :author_id) - }.tap do |h| - if time_period.present? - h[:merge_requests_users] = merge_requests_users(time_period) - h.merge!(action_monthly_active_users(time_period)) - end - end + } end # rubocop: enable CodeReuse/ActiveRecord @@ -527,7 +467,6 @@ module Gitlab # Omitted because no user, creator or author associated: `boards`, `labels`, `milestones`, `uploads` # Omitted because too expensive: `epics_deepest_relationship_level` - # Omitted because of encrypted properties: `projects_jira_cloud_active`, `projects_jira_server_active` # rubocop: disable CodeReuse/ActiveRecord def usage_activity_by_stage_plan(time_period) time_frame = metric_time_period(time_period) @@ -582,17 +521,6 @@ module Gitlab {} end - def action_monthly_active_users(time_period) - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } - - { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) } - } - end - def with_metadata result = nil error = nil @@ -717,7 +645,6 @@ module Gitlab time_frame = metric_time_period(time_period) counters = { gitlab_project: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab_project' }), - gitlab: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'gitlab' }), github: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'github' }), bitbucket: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket' }), bitbucket_server: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'bitbucket_server' }), @@ -737,7 +664,6 @@ module Gitlab { jira: count(::JiraImportState.where(time_period)), # rubocop: disable CodeReuse/ActiveRecord fogbugz: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'fogbugz' }), - phabricator: add_metric('CountImportedProjectsMetric', time_frame: time_frame, options: { import_type: 'phabricator' }), csv: count(::Issues::CsvImport.where(time_period)) # rubocop: disable CodeReuse/ActiveRecord } end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 7f6d67e01c7..97091ff975b 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -2,7 +2,7 @@ module Gitlab::UsageDataCounters class CiTemplateUniqueCounter - REDIS_SLOT = 'ci_templates' + PREFIX = 'ci_templates' KNOWN_EVENTS_FILE_PATH = File.expand_path('known_events/ci_templates.yml', __dir__) class << self @@ -28,7 +28,7 @@ module Gitlab::UsageDataCounters def ci_template_event_name(template_name, config_source) prefix = 'implicit_' if config_source.to_s == 'auto_devops_source' - "p_#{REDIS_SLOT}_#{prefix}#{template_to_event_name(template_name)}" + "p_#{PREFIX}_#{prefix}#{template_to_event_name(template_name)}" end def expand_template_name(template_name) diff --git a/lib/gitlab/usage_data_counters/container_registry_event_counter.rb b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb new file mode 100644 index 00000000000..5d54bb18443 --- /dev/null +++ b/lib/gitlab/usage_data_counters/container_registry_event_counter.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module UsageDataCounters + class ContainerRegistryEventCounter < BaseCounter + KNOWN_EVENTS = %w[i_container_registry_delete_manifest].freeze + PREFIX = 'container_registry_events' + end + end +end diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index f7ddc53f50d..129bf77c7f0 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -7,6 +7,7 @@ - i_package_conan_push_package - i_package_debian_delete_package - i_package_debian_pull_package +- i_package_debian_push_package - i_package_delete_package - i_package_delete_package_by_deploy_token - i_package_delete_package_by_guest diff --git a/lib/gitlab/usage_data_counters/editor_unique_counter.rb b/lib/gitlab/usage_data_counters/editor_unique_counter.rb index 2aebc1b8813..4e4a01ed301 100644 --- a/lib/gitlab/usage_data_counters/editor_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/editor_unique_counter.rb @@ -38,18 +38,16 @@ module Gitlab def track_unique_action(event_name, author, time, project = nil) return unless author - if Feature.enabled?(:route_hll_to_snowplow_phase2) - Gitlab::Tracking.event( - name, - 'ide_edit', - property: event_name.to_s, - project: project, - namespace: project&.namespace, - user: author, - label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', - context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] - ) - end + Gitlab::Tracking.event( + name, + 'ide_edit', + property: event_name.to_s, + project: project, + namespace: project&.namespace, + user: author, + label: 'usage_activity_by_stage_monthly.create.action_monthly_active_users_ide_edit', + context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] + ) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id, time: time) end diff --git a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb index 8a57a0331b8..b30c4b675f9 100644 --- a/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/gitlab_cli_activity_unique_counter.rb @@ -4,7 +4,9 @@ module Gitlab module UsageDataCounters module GitLabCliActivityUniqueCounter GITLAB_CLI_API_REQUEST_ACTION = 'i_code_review_user_gitlab_cli_api_request' - GITLAB_CLI_USER_AGENT_REGEX = /GitLab\sCLI$/.freeze + + # This regex will match to user agents ending with GitLab CLI or starting with glab/v" + GITLAB_CLI_USER_AGENT_REGEX = %r{(GitLab\sCLI$|^glab/v)}.freeze class << self def track_api_request_when_trackable(user_agent:, user:) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index b809e6c4e42..7ddd496cd85 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -5,26 +5,17 @@ module Gitlab module HLLRedisCounter DEFAULT_WEEKLY_KEY_EXPIRY_LENGTH = 6.weeks DEFAULT_DAILY_KEY_EXPIRY_LENGTH = 29.days - DEFAULT_REDIS_SLOT = '' + REDIS_SLOT = 'hll_counters' EventError = Class.new(StandardError) UnknownEvent = Class.new(EventError) UnknownAggregation = Class.new(EventError) AggregationMismatch = Class.new(EventError) - SlotMismatch = Class.new(EventError) - CategoryMismatch = Class.new(EventError) InvalidContext = Class.new(EventError) KNOWN_EVENTS_PATH = File.expand_path('known_events/*.yml', __dir__) ALLOWED_AGGREGATIONS = %i(daily weekly).freeze - CATEGORIES_FOR_TOTALS = %w[ - compliance - error_tracking - ide_edit - pipeline_authoring - ].freeze - # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # @@ -33,10 +24,7 @@ module Gitlab # Event example: # # - name: g_compliance_dashboard # Unique event name - # redis_slot: compliance # Optional slot name, if not defined it will use name as a slot, used for totals - # category: compliance # Group events in categories - # aggregation: daily # Aggregation level, keys are stored daily or weekly - # feature_flag: # The event feature flag + # aggregation: weekly # Aggregation level, keys are stored weekly # # Usage: # @@ -76,23 +64,11 @@ module Gitlab # context - Event context, plan level tracking. Available if set when tracking. def unique_events(event_names:, start_date:, end_date:, context: '') count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date, context: context) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) - raise CategoryMismatch, events unless events_in_same_category?(events) raise AggregationMismatch, events unless events_same_aggregation?(events) raise InvalidContext if context.present? && !context.in?(valid_context_list) end end - def categories - @categories ||= known_events.map { |event| event[:category] }.uniq - end - - # @param category [String] the category name - # @return [Array<String>] list of event names for given category - def events_for_category(category) - known_events.select { |event| event[:category] == category.to_s }.map { |event| event[:name] } - end - def known_event?(event_name) event_for(event_name).present? end @@ -103,7 +79,6 @@ module Gitlab def calculate_events_union(event_names:, start_date:, end_date:) count_unique_events(event_names: event_names, start_date: start_date, end_date: end_date) do |events| - raise SlotMismatch, events unless events_in_same_slot?(events) raise AggregationMismatch, events unless events_same_aggregation?(events) end end @@ -117,9 +92,15 @@ module Gitlab Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? return if event.blank? - return unless feature_enabled?(event) + return unless Feature.enabled?(:redis_hll_tracking, type: :ops) + + if event[:aggregation].to_sym == :daily + weekly_event = event.dup.tap { |e| e['aggregation'] = 'weekly' } + Gitlab::Redis::HLL.add(key: redis_key(weekly_event, time, context), value: values, expiry: expiry(weekly_event)) + end Gitlab::Redis::HLL.add(key: redis_key(event, time, context), value: values, expiry: expiry(event)) + rescue StandardError => e # Ignore any exceptions unless is dev or test env # The application flow should not be blocked by erros in tracking @@ -138,6 +119,11 @@ module Gitlab aggregation = events.first[:aggregation] + if Feature.disabled?(:revert_daily_hll_events_to_weekly_aggregation) + aggregation = :weekly + events.each { |e| e[:aggregation] = :weekly } + end + keys = keys_for_aggregation(aggregation, events: events, start_date: start_date, end_date: end_date, context: context) return FALLBACK unless keys.any? @@ -145,21 +131,6 @@ module Gitlab redis_usage_data { Gitlab::Redis::HLL.count(keys: keys) } end - def feature_enabled?(event) - return true if event[:feature_flag].blank? - - Feature.enabled?(event[:feature_flag]) && Feature.enabled?(:redis_hll_tracking, type: :ops) - end - - # Allow to add totals for events that are in the same redis slot, category and have the same aggregation level - # and if there are more than 1 event - def eligible_for_totals?(events_names) - return false if events_names.size <= 1 - - events = events_for(events_names) - events_in_same_slot?(events) && events_in_same_category?(events) && events_same_aggregation?(events) - end - def keys_for_aggregation(aggregation, events:, start_date:, end_date:, context: '') if aggregation.to_sym == :daily daily_redis_keys(events: events, start_date: start_date, end_date: end_date, context: context) @@ -182,20 +153,6 @@ module Gitlab known_events.map { |event| event[:name] } end - def events_in_same_slot?(events) - # if we check one event then redis_slot is only one to check - return false if events.empty? - return true if events.size == 1 - - slot = events.first[:redis_slot] - events.all? { |event| event[:redis_slot].present? && event[:redis_slot] == slot } - end - - def events_in_same_category?(events) - category = events.first[:category] - events.all? { |event| event[:category] == category } - end - def events_same_aggregation?(events) aggregation = events.first[:aggregation] events.all? { |event| event[:aggregation] == aggregation } @@ -213,30 +170,19 @@ module Gitlab known_events.select { |event| event_names.include?(event[:name]) } end - def redis_slot(event) - event[:redis_slot] || DEFAULT_REDIS_SLOT - end - # Compose the key in order to store events daily or weekly def redis_key(event, time, context = '') raise UnknownEvent, "Unknown event #{event[:name]}" unless known_events_names.include?(event[:name].to_s) + + # ToDo: remove during https://gitlab.com/groups/gitlab-org/-/epics/9542 cleanup raise UnknownAggregation, "Use :daily or :weekly aggregation" unless ALLOWED_AGGREGATIONS.include?(event[:aggregation].to_sym) - key = apply_slot(event) + key = "{#{REDIS_SLOT}}_#{event[:name]}" key = apply_time_aggregation(key, time, event) key = "#{context}_#{key}" if context.present? key end - def apply_slot(event) - slot = redis_slot(event) - if slot.present? - event[:name].to_s.gsub(slot, "{#{slot}}") - else - "{#{event[:name]}}" - end - end - def apply_time_aggregation(key, time, event) if event[:aggregation].to_sym == :daily year_day = time.strftime('%G-%j') diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index a59ea36961d..31f090e0f51 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -37,8 +37,8 @@ module Gitlab ISSUE_DESIGN_COMMENT_REMOVED = 'g_project_management_issue_design_comments_removed' class << self - def track_issue_created_action(author:, project:) - track_snowplow_action(ISSUE_CREATED, author, project) + def track_issue_created_action(author:, namespace:) + track_snowplow_action(ISSUE_CREATED, author, namespace) track_unique_action(ISSUE_CREATED, author) end @@ -179,8 +179,16 @@ module Gitlab private - def track_snowplow_action(event_name, author, project) - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) + def track_snowplow_action(event_name, author, container) + namespace, project = case container + when Project + [container.namespace, container] + when Namespaces::ProjectNamespace + [container.parent, container.project] + else + [container, nil] + end + return unless author Gitlab::Tracking.event( @@ -189,7 +197,7 @@ module Gitlab label: ISSUE_LABEL, property: event_name, project: project, - namespace: project.namespace, + namespace: namespace, user: author, context: [Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, event: event_name).to_context] ) diff --git a/lib/gitlab/usage_data_counters/known_events/analytics.yml b/lib/gitlab/usage_data_counters/known_events/analytics.yml index 85524c766ca..0b30308b552 100644 --- a/lib/gitlab/usage_data_counters/known_events/analytics.yml +++ b/lib/gitlab/usage_data_counters/known_events/analytics.yml @@ -1,52 +1,26 @@ - name: users_viewing_analytics_group_devops_adoption - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_adoption - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_dev_ops_score - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_instance_statistics - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_pipelines - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_valuestream - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_repo - category: analytics - redis_slot: analytics aggregation: weekly - name: i_analytics_cohorts - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_pipelines - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_deployment_frequency - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_lead_time - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_time_to_restore_service - category: analytics - redis_slot: analytics aggregation: weekly - name: p_analytics_ci_cd_change_failure_rate - category: analytics - redis_slot: analytics aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index b13e3d631c7..f685f0d65d9 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -4,602 +4,306 @@ # Do not edit it manually! --- - name: p_ci_templates_terraform_base_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_base - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_nodejs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_openshift - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_auto_devops - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_bash - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_rust - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_elixir - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_clojure - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_crystal - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_getting_started - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_load_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_accessibility - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_failfast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_verify_browser_performance_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_grails - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_runner_validation - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_scan - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_on_demand_api_scan - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_coverage_fuzzing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_secure_binaries - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast_api_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_container_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_api_fuzzing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_dast - category: ci_templates - redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_api_discovery aggregation: weekly - name: p_ci_templates_security_fortify_fod_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_security_sast_iac_latest - category: ci_templates - redis_slot: ci_templates + aggregation: weekly +- name: p_ci_templates_security_bas_latest aggregation: weekly - name: p_ci_templates_qualys_iac_security - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ios_fastlane - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_composer - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_c - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_python - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_fastlane - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_django - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_maven - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_liquibase - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_flutter - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_branch_pipelines - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_workflows_mergerequest_pipelines - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_laravel - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_kaniko - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_php - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_packer - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_themekit - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_katalon - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_mono - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_go - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_scala - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_latex - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_android - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_indeni_cloudrail - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_matlab - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_cf_provision_and_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_aws_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_gradle - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_chef - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_load_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_intelligence - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_license_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_container_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_test - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_secret_detection_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_deploy_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_browser_performance_testing_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_cf_provision - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_build_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_jobs_sast_iac_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_latest - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_swift - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jekyll - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_harp - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_octopress - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_brunch - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_doxygen - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hyde - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_lektor - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jbake - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hexo - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_middleman - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_hugo - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_pelican - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_nanoc - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_swaggerui - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_jigsaw - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_metalsmith - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_gatsby - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_pages_html - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dart - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_docker - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_julia - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_npm - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_dotnet_core - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_5_minute_production_app - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_ruby - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_auto_devops - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_browser_performance_testing - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_build - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_intelligence - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_code_quality - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dast_default_branch_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ec2 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy_ecs - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_helm_2to3 - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_jobs_test - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_container_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_dependency_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_license_scanning - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_sast - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_implicit_security_secret_detection - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module_base - category: ci_templates - redis_slot: ci_templates aggregation: weekly - name: p_ci_templates_terraform_module - category: ci_templates - redis_slot: ci_templates aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ci_users.yml b/lib/gitlab/usage_data_counters/known_events/ci_users.yml index b012d61eef5..49757c6e672 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_users.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_users.yml @@ -1,10 +1,4 @@ - name: ci_users_executing_deployment_job - category: ci_users - redis_slot: ci_users aggregation: weekly - feature_flag: - name: ci_users_executing_verify_environment_job - category: ci_users - redis_slot: ci_users aggregation: weekly - feature_flag: diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index 3bb6655d762..db0c0653f63 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,457 +1,233 @@ --- - name: i_code_review_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_mr_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_single_file_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_mr_single_file_diffs - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_toggled_task_item_status - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_create_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_close_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_reopen_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approve_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unapprove_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_thread - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unresolve_thread - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_edit_mr_title - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_edit_mr_desc - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_merge_mr - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_edit_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_remove_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_review_note - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_publish_review - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_edit_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_remove_multiline_mr_comment - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_add_suggestion - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_apply_suggestion - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_assigned - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_marked_as_draft - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_unmarked_as_draft - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_review_requested - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_added - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_deleted - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_approval_rule_edited - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_vs_code_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_jetbrains_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_gitlab_cli_api_request - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_create_mr_from_issue - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_locked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_mr_discussion_unlocked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_time_estimate_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_time_spent_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_assignees_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_reviewers_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_milestone_changed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_labels_changed - redis_slot: code_review - category: code_review aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_single_file_mode_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_file_browser_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_click_whitespace_setting - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_view_inline - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_view_parallel - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_file_browser_tree_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_file_browser_list_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_show_whitespace - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_hide_whitespace - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_single_file - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_diff_multiple_files - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_load_conflict_ui - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_conflict - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_searches_diff - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_total_suggestions_applied - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_total_suggestions_added - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_user_resolve_thread_in_issue - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_widget_nothing_merge_click_new_file - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_delete_branch - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_click_revert - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_click_cherry_pick - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_submit_revert_modal - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_post_merge_submit_cherry_pick_modal - redis_slot: code_review - category: code_review aggregation: weekly # MR Widget Extensions ## Test Summary - name: i_code_review_merge_request_widget_test_summary_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_test_summary_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Accessibility - name: i_code_review_merge_request_widget_accessibility_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_accessibility_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Code Quality - name: i_code_review_merge_request_widget_code_quality_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_code_quality_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Terraform - name: i_code_review_merge_request_widget_terraform_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_terraform_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_submit_review_approve - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_submit_review_comment - redis_slot: code_review - category: code_review aggregation: weekly ## License Compliance - name: i_code_review_merge_request_widget_license_compliance_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_license_compliance_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly ## Security Reports - name: i_code_review_merge_request_widget_security_reports_view - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_full_report_clicked - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_success - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_warning - redis_slot: code_review - category: code_review aggregation: weekly - name: i_code_review_merge_request_widget_security_reports_expand_failed - redis_slot: code_review - category: code_review aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index ae15530f0d0..0583d85c3cc 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -1,313 +1,167 @@ --- # Compliance category - name: g_edit_by_web_ide - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_sfe - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_snippet_ide - category: ide_edit - redis_slot: edit aggregation: daily - name: g_edit_by_live_preview - category: ide_edit - redis_slot: edit aggregation: daily - name: i_search_total - category: search - redis_slot: search aggregation: weekly - name: wiki_action - category: source_code aggregation: daily - name: design_action - category: source_code aggregation: daily - name: project_action - category: source_code aggregation: daily - name: git_write_action - category: source_code aggregation: daily - name: merge_request_action - category: source_code aggregation: daily - name: i_source_code_code_intelligence - redis_slot: source_code - category: source_code aggregation: daily # Incident management - name: incident_management_alert_status_changed - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_alert_assigned - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_alert_todo - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_created - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_reopened - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_closed - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_assigned - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_todo - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_comment - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_zoom_meeting - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_relate - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_unrelate - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_incident_change_confidential - redis_slot: incident_management - category: incident_management aggregation: weekly # Incident management timeline events - name: incident_management_timeline_event_created - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_timeline_event_edited - redis_slot: incident_management - category: incident_management aggregation: weekly - name: incident_management_timeline_event_deleted - redis_slot: incident_management - category: incident_management aggregation: weekly # Incident management alerts - name: incident_management_alert_create_incident - redis_slot: incident_management - category: incident_management_alerts aggregation: weekly # Testing category - name: i_testing_test_case_parsed - category: testing - redis_slot: testing - aggregation: weekly -- name: i_testing_summary_widget_total - category: testing - redis_slot: testing aggregation: weekly - name: i_testing_test_report_uploaded - category: testing - redis_slot: testing aggregation: weekly - name: i_testing_coverage_report_uploaded - category: testing - redis_slot: testing aggregation: weekly # Project Management group - name: g_project_management_issue_title_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_description_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_assignee_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_confidential - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_made_visible - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_created - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_closed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_reopened - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_label_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_milestone_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_cross_referenced - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_moved - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_related - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_unrelated - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_marked_as_duplicate - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_locked - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_unlocked - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_added - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_modified - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_designs_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_due_date_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_design_comments_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_estimate_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_time_spent_changed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_added - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_edited - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_comment_removed - category: issues_edit - redis_slot: project_management aggregation: daily - name: g_project_management_issue_cloned - category: issues_edit - redis_slot: project_management aggregation: daily # Runner group - name: g_runner_fleet_read_jobs_statistics - category: runner - redis_slot: runner aggregation: weekly # Secrets Management - name: i_snippets_show - category: snippets - redis_slot: snippets aggregation: weekly # Terraform - name: p_terraform_state_api_unique_users - category: terraform - redis_slot: terraform aggregation: weekly # Pipeline Authoring group +- name: ci_interpolation_users + aggregation: weekly - name: o_pipeline_authoring_unique_users_committing_ciconfigfile - category: pipeline_authoring - redis_slot: pipeline_authoring aggregation: weekly - name: o_pipeline_authoring_unique_users_pushing_mr_ciconfigfile - category: pipeline_authoring - redis_slot: pipeline_authoring aggregation: weekly - name: i_ci_secrets_management_id_tokens_build_created - category: ci_secrets_management - redis_slot: ci_secrets_management aggregation: weekly # Merge request widgets - name: users_expanding_secure_security_report - redis_slot: secure - category: secure aggregation: weekly - name: users_expanding_testing_code_quality_report - redis_slot: testing - category: testing aggregation: weekly - name: users_expanding_testing_accessibility_report - redis_slot: testing - category: testing aggregation: weekly - name: users_expanding_testing_license_compliance_report - redis_slot: testing - category: testing aggregation: weekly - name: users_visiting_testing_license_compliance_full_report - redis_slot: testing - category: testing aggregation: weekly - name: users_visiting_testing_manage_license_compliance - redis_slot: testing - category: testing aggregation: weekly - name: users_clicking_license_testing_visiting_external_website - redis_slot: testing - category: testing aggregation: weekly # Geo group - name: g_geo_proxied_requests - category: geo - redis_slot: geo aggregation: daily # Manage - name: unique_active_user - category: manage aggregation: weekly # Environments page - name: users_visiting_environments_pages - category: environments - redis_slot: users aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml index e8b14de1769..aa0f9965fa7 100644 --- a/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/container_registry_events.yml @@ -1,22 +1,11 @@ --- - name: i_container_registry_push_tag_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_delete_tag_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_push_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_delete_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry - name: i_container_registry_create_repository_user - category: user_container_registry aggregation: weekly - redis_slot: container_registry -
\ No newline at end of file diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index 7f7c9166086..6e4a893d19a 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -1,46 +1,24 @@ --- # Ecosystem category - name: i_ecosystem_jira_service_close_issue - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_jira_service_cross_reference - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_issue_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_push_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_deployment_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_wiki_page_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_merge_request_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_note_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_tag_push_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_note_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly - name: i_ecosystem_slack_service_confidential_issue_notification - category: ecosystem - redis_slot: ecosystem aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml index d80b711f8eb..ebfd1b274f9 100644 --- a/lib/gitlab/usage_data_counters/known_events/error_tracking.yml +++ b/lib/gitlab/usage_data_counters/known_events/error_tracking.yml @@ -1,9 +1,5 @@ --- - name: error_tracking_view_details - category: error_tracking - redis_slot: error_tracking aggregation: weekly - name: error_tracking_view_list - category: error_tracking - redis_slot: error_tracking aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/importer_events.yml b/lib/gitlab/usage_data_counters/known_events/importer_events.yml index c84d756a013..abbd83a012b 100644 --- a/lib/gitlab/usage_data_counters/known_events/importer_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/importer_events.yml @@ -1,14 +1,13 @@ --- # Importer events - name: github_import_project_start - category: importer - redis_slot: import aggregation: weekly - name: github_import_project_success - category: importer - redis_slot: import aggregation: weekly - name: github_import_project_failure - category: importer - redis_slot: import aggregation: weekly +- name: github_import_project_cancelled + aggregation: weekly +- name: github_import_project_partially_completed + aggregation: weekly + diff --git a/lib/gitlab/usage_data_counters/known_events/integrations.yml b/lib/gitlab/usage_data_counters/known_events/integrations.yml new file mode 100644 index 00000000000..4a83581e9f0 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/integrations.yml @@ -0,0 +1,18 @@ +- name: i_integrations_gitlab_for_slack_app_issue_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_push_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_deployment_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_wiki_page_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_merge_request_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_note_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_tag_push_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_confidential_note_notification + aggregation: weekly +- name: i_integrations_gitlab_for_slack_app_confidential_issue_notification + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index 966e6c584c7..b3d1c51c0e7 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -1,4 +1,2 @@ - name: agent_users_using_ci_tunnel - category: kubernetes_agent - redis_slot: agent aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index ef8d02fa365..fa99798cde0 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -1,89 +1,49 @@ --- - name: i_package_composer_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_composer_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_conan_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_conan_user - category: user_packages aggregation: weekly - redis_slot: package +- name: i_package_debian_deploy_token + aggregation: weekly +- name: i_package_debian_user + aggregation: weekly - name: i_package_generic_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_generic_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_helm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_helm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_maven_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_maven_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_npm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_npm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_nuget_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_nuget_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_pypi_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_pypi_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rubygems_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_rubygems_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_terraform_module_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package - name: i_package_terraform_module_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rpm_user - category: user_packages aggregation: weekly - redis_slot: package - name: i_package_rpm_deploy_token - category: deploy_token_packages aggregation: weekly - redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/product_analytics.yml b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml new file mode 100644 index 00000000000..5a791c4b3c2 --- /dev/null +++ b/lib/gitlab/usage_data_counters/known_events/product_analytics.yml @@ -0,0 +1,4 @@ +- name: project_created_analytics_dashboard + aggregation: weekly +- name: project_initialized_product_analytics + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index 69b348b9a22..136d284f462 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -1,253 +1,135 @@ --- - name: i_quickactions_assign_multiple - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_approve - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unapprove - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_single - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_self - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_assign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_award - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_board_move - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_clone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_close - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_confidential - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_merge_request - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_copy_metadata_issue - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_create_merge_request - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_done - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_draft - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_due - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_duplicate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_estimate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_label - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_lock - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_merge - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_milestone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_move - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_promote_to_incident - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_timeline - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_ready - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reassign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_rebase - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_relabel - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_relate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_due_date - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_estimate - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_milestone - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_time_spent - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_zoom - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_reopen - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_severity - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_shrug - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_subtract - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_spend_add - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_submit_review - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_subscribe - category: quickactions - redis_slot: quickactions + aggregation: weekly +- name: i_quickactions_summarize_diff aggregation: weekly - name: i_quickactions_tableflip - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_tag - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_target_branch - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_title - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_todo - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_specific - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_all - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unassign_reviewer - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_specific - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlabel_all - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unlock - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_unsubscribe - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_wip - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_zoom - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_link - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_single - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_invite_email_multiple - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_add_contacts - category: quickactions - redis_slot: quickactions aggregation: weekly - name: i_quickactions_remove_contacts - category: quickactions - redis_slot: quickactions + aggregation: weekly +- name: i_quickactions_type + aggregation: weekly +- name: i_quickactions_blocked_by + aggregation: weekly +- name: i_quickactions_blocks aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index d088b6d7e5a..a6e5b9e1af5 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -1,42 +1,21 @@ --- - name: users_updating_work_item_title - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_creating_work_items - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_dates - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_labels - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_milestone - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_work_item_iteration # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity - name: users_updating_weight_estimate # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. # It will report 0 for CE instances and should not be used with 'AND' aggregators. - category: work_items - redis_slot: users aggregation: weekly - feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index c8768164710..fceeacb60ca 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -68,8 +68,6 @@ module Gitlab track_unique_action_by_merge_request(MR_CREATE_ACTION, merge_request) project = merge_request.target_project - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) - Gitlab::Tracking.event( name, :create, @@ -99,8 +97,6 @@ module Gitlab track_unique_action_by_user(MR_APPROVE_ACTION, user) project = merge_request.target_project - return unless Feature.enabled?(:route_hll_to_snowplow_phase2, project.namespace) - Gitlab::Tracking.event( name, :approve, diff --git a/lib/gitlab/usage_data_counters/track_unique_events.rb b/lib/gitlab/usage_data_counters/track_unique_events.rb deleted file mode 100644 index 20da9665876..00000000000 --- a/lib/gitlab/usage_data_counters/track_unique_events.rb +++ /dev/null @@ -1,81 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageDataCounters - module TrackUniqueEvents - WIKI_ACTION = :wiki_action - DESIGN_ACTION = :design_action - PUSH_ACTION = :project_action - MERGE_REQUEST_ACTION = :merge_request_action - - GIT_WRITE_ACTIONS = [WIKI_ACTION, DESIGN_ACTION, PUSH_ACTION].freeze - GIT_WRITE_ACTION = :git_write_action - - ACTION_TRANSFORMATIONS = HashWithIndifferentAccess.new({ - wiki: { - created: WIKI_ACTION, - updated: WIKI_ACTION, - destroyed: WIKI_ACTION - }, - design: { - created: DESIGN_ACTION, - updated: DESIGN_ACTION, - destroyed: DESIGN_ACTION - }, - project: { - pushed: PUSH_ACTION - }, - merge_request: { - closed: MERGE_REQUEST_ACTION, - merged: MERGE_REQUEST_ACTION, - created: MERGE_REQUEST_ACTION, - commented: MERGE_REQUEST_ACTION - } - }).freeze - - class << self - def track_event(event_action:, event_target:, author_id:, time: Time.zone.now) - return unless valid_target?(event_target) - return unless valid_action?(event_action) - - transformed_target = transform_target(event_target) - transformed_action = transform_action(event_action, transformed_target) - - return unless Gitlab::UsageDataCounters::HLLRedisCounter.known_event?(transformed_action.to_s) - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(transformed_action.to_s, values: author_id, time: time) - - track_git_write_action(author_id, transformed_action, time) - end - - def count_unique_events(event_action:, date_from:, date_to:) - Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: event_action.to_s, start_date: date_from, end_date: date_to) - end - - private - - def transform_action(event_action, event_target) - ACTION_TRANSFORMATIONS.dig(event_target, event_action) || event_action - end - - def transform_target(event_target) - Event::TARGET_TYPES.key(event_target) - end - - def valid_target?(target) - Event::TARGET_TYPES.value?(target) - end - - def valid_action?(action) - Event.actions.key?(action) - end - - def track_git_write_action(author_id, transformed_action, time) - return unless GIT_WRITE_ACTIONS.include?(transformed_action) - - Gitlab::UsageDataCounters::HLLRedisCounter.track_event(GIT_WRITE_ACTION, values: author_id, time: time) - end - end - end - end -end diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index b99c9ebb24f..9de575d8567 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -33,7 +33,7 @@ module Gitlab private def track_unique_action(action, author) - return unless author + return unless author && Feature.enabled?(:track_work_items_activity) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(action, values: author.id) end diff --git a/lib/gitlab/usage_data_metrics.rb b/lib/gitlab/usage_data_metrics.rb index 48f695d5db1..8948d621d3c 100644 --- a/lib/gitlab/usage_data_metrics.rb +++ b/lib/gitlab/usage_data_metrics.rb @@ -8,10 +8,6 @@ module Gitlab build_payload(:with_value) end - def suggested_names - build_payload(:with_suggested_name) - end - private def build_payload(method_symbol) diff --git a/lib/gitlab/usage_data_non_sql_metrics.rb b/lib/gitlab/usage_data_non_sql_metrics.rb index 79d4b45a1ce..71386a58ba7 100644 --- a/lib/gitlab/usage_data_non_sql_metrics.rb +++ b/lib/gitlab/usage_data_non_sql_metrics.rb @@ -40,13 +40,6 @@ module Gitlab def minimum_id(model, column = nil) end - - def jira_integration_data - { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - end end end end diff --git a/lib/gitlab/usage_data_queries.rb b/lib/gitlab/usage_data_queries.rb index 3a163e5dde9..534a08cad9a 100644 --- a/lib/gitlab/usage_data_queries.rb +++ b/lib/gitlab/usage_data_queries.rb @@ -68,13 +68,6 @@ module Gitlab end end - def jira_integration_data - { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - end - def topology_usage_data { duration_s: 0, diff --git a/lib/gitlab/utils/email.rb b/lib/gitlab/utils/email.rb index c65d7165263..5eb57a66c63 100644 --- a/lib/gitlab/utils/email.rb +++ b/lib/gitlab/utils/email.rb @@ -8,19 +8,81 @@ module Gitlab # Replaces most visible characters with * to obfuscate an email address # deform adds a fix number of * to ensure the address cannot be guessed. Also obfuscates TLD with ** def obfuscated_email(email, deform: false) - regex = ::Gitlab::UntrustedRegexp.new('^(..?)(.*)(@.?)(.*)(\..+)$') - match = regex.match(email) - return email unless match - - if deform - # Ensure we can show two characters for the username, even if the username has - # only one character. Boring solution is to just duplicate the character. - email_start = match[1] - email_start += email_start if email_start.length == 1 - - email_start + '*' * 5 + match[3] + '*' * 5 + "#{match[5][0..1]}**" - else - match[1] + '*' * (match[2] || '').length + match[3] + '*' * (match[4] || '').length + match[5] + return email if email.empty? + + masker_class = deform ? Deform : Symmetrical + masker_class.new(email).masked + end + + class Masker + attr_reader :local_part, :sub_domain, :toplevel_domain, :at, :dot + + def initialize(original) + @original = original + @local_part, @at, domain = original.rpartition('@') + @sub_domain, @dot, @toplevel_domain = domain.rpartition('.') + + @at = nil if @at.empty? + @dot = nil if @dot.empty? + end + + def masked + masked = [ + local_part, + at, + sub_domain, + dot, + toplevel_domain + ].compact.join('') + + masked = mask(@original, visible_length: 1) if masked == @original + + masked + end + + private + + def mask(plain, visible_length:, star_length: nil) + return if plain.empty? + return plain if visible_length < 0 + + plain = enlarge_if_needed(plain, visible_length) + + star_length = plain.length - visible_length if star_length.nil? + + first = plain[0, visible_length] + stars = '*' * star_length + + "#{first}#{stars}" + end + + def enlarge_if_needed(string, min) + string.ljust(min, string.first) + end + end + + class Symmetrical < Masker + def local_part + mask(@local_part, visible_length: 2) + end + + def sub_domain + mask(@sub_domain, visible_length: 1) + end + end + + # Implements https://design.gitlab.com/usability/obfuscation#email-addresses + class Deform < Masker + def local_part + mask(@local_part, visible_length: 2, star_length: 5) + end + + def sub_domain + mask(@sub_domain, visible_length: 1, star_length: 5) + end + + def toplevel_domain + mask(@toplevel_domain, visible_length: 1, star_length: 2) end end end diff --git a/lib/gitlab/utils/error_message.rb b/lib/gitlab/utils/error_message.rb new file mode 100644 index 00000000000..cd9af95bfc3 --- /dev/null +++ b/lib/gitlab/utils/error_message.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + module ErrorMessage + extend self + + UF_ERROR_PREFIX = 'UF' + + def to_user_facing(message) + prefixed_error_message(message, UF_ERROR_PREFIX) + end + + def prefixed_error_message(message, prefix) + "#{prefix} #{message}" + end + end + end +end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 7f43e25e50d..1d02bcbb2d2 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true -require_dependency 'gitlab/utils' -require_dependency 'gitlab/environment' +require_relative '../utils' +require_relative '../environment' module Gitlab module Utils diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index eb44b7ddd95..2b3841b8f09 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -35,6 +35,27 @@ module Gitlab end end + # Works the same way as "strong_memoize" but takes + # a second argument - expire_in. This allows invalidate + # the data after specified number of seconds + def strong_memoize_with_expiration(name, expire_in) + key = ivar(name) + expiration_key = "#{key}_expired_at" + + if instance_variable_defined?(expiration_key) + expire_at = instance_variable_get(expiration_key) + clear_memoization(name) if Time.current > expire_at + end + + if instance_variable_defined?(key) + instance_variable_get(key) + else + value = instance_variable_set(key, yield) + instance_variable_set(expiration_key, Time.current + expire_in) + value + end + end + def strong_memoize_with(name, *args) container = strong_memoize(name) { {} } diff --git a/lib/gitlab/utils/uniquify.rb b/lib/gitlab/utils/uniquify.rb new file mode 100644 index 00000000000..b5908d18103 --- /dev/null +++ b/lib/gitlab/utils/uniquify.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +# Uniquify +# +# Return a version of the given 'base' string that is unique +# by appending a counter to it. Uniqueness is determined by +# repeated calls to the passed block. +# +# You can pass an initial value for the counter, if not given +# counting starts from 1. +# +# If `base` is a function/proc, we expect that calling it with a +# candidate counter returns a string to test/return. + +module Gitlab + module Utils + class Uniquify + def initialize(counter = nil) + @counter = counter + end + + def string(base) + @base = base + + increment_counter! while yield(base_string) + base_string + end + + private + + def base_string + if @base.respond_to?(:call) + @base.call(@counter) + else + "#{@base}#{@counter}" + end + end + + def increment_counter! + @counter ||= 0 + @counter += 1 + end + end + end +end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index fab8617bcda..4106084b301 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -255,33 +255,6 @@ module Gitlab end end - # rubocop: disable UsageData/LargeTable: - def jira_integration_data - with_metadata do - data = { - projects_jira_server_active: 0, - projects_jira_cloud_active: 0 - } - - # rubocop: disable CodeReuse/ActiveRecord - ::Integrations::Jira.active.includes(:jira_tracker_data).find_in_batches(batch_size: 100) do |services| - counts = services.group_by do |service| - # TODO: Simplify as part of https://gitlab.com/gitlab-org/gitlab/issues/29404 - service_url = service.data_fields&.url || (service.properties && service.properties['url']) - service_url&.include?('.atlassian.net') ? :cloud : :server - end - - data[:projects_jira_server_active] += counts[:server].size if counts[:server] - data[:projects_jira_cloud_active] += counts[:cloud].size if counts[:cloud] - end - - data - end - end - - # rubocop: enable CodeReuse/ActiveRecord - # rubocop: enable UsageData/LargeTable: - def minimum_id(model, column = nil) key = :"#{model.name.downcase.gsub('::', '_')}_minimum_id" column_to_read = column || :id diff --git a/lib/gitlab/utils/username_and_email_generator.rb b/lib/gitlab/utils/username_and_email_generator.rb new file mode 100644 index 00000000000..38c9bb7050d --- /dev/null +++ b/lib/gitlab/utils/username_and_email_generator.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'securerandom' + +module Gitlab + module Utils + class UsernameAndEmailGenerator + include Gitlab::Utils::StrongMemoize + + def initialize(username_prefix:, email_domain: Gitlab.config.gitlab.host) + @username_prefix = username_prefix + @email_domain = email_domain + end + + def username + uniquify.string(->(counter) { Kernel.sprintf(username_pattern, counter) }) do |suggested_username| + ::Namespace.by_path(suggested_username) || ::User.find_by_any_email(email_for(suggested_username)) + end + end + strong_memoize_attr :username + + def email + email_for(username) + end + strong_memoize_attr :email + + private + + def username_pattern + "#{@username_prefix}_#{SecureRandom.hex(16)}%s" + end + + def email_for(name) + "#{name}@#{@email_domain}" + end + + def uniquify + Gitlab::Utils::Uniquify.new + end + end + end +end diff --git a/lib/gitlab/verify/batch_verifier.rb b/lib/gitlab/verify/batch_verifier.rb index 71d106db742..bff95743dbd 100644 --- a/lib/gitlab/verify/batch_verifier.rb +++ b/lib/gitlab/verify/batch_verifier.rb @@ -34,7 +34,7 @@ module Gitlab private def run_batch_for(batch) - batch.map { |upload| verify(upload) }.compact.to_h + batch.filter_map { |upload| verify(upload) }.to_h end def verify(object) diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 02418c45e73..79131404465 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -156,11 +156,14 @@ module Gitlab ] end - def send_url(url, allow_redirects: false) + def send_url(url, allow_redirects: false, method: 'GET', body: nil, headers: nil) params = { 'URL' => url, - 'AllowRedirects' => allow_redirects - } + 'AllowRedirects' => allow_redirects, + 'Body' => body.to_s, + 'Header' => headers, + 'Method' => method + }.compact [ SEND_DATA_HEADER, diff --git a/lib/gitlab/x509/signature.rb b/lib/gitlab/x509/signature.rb index d6bbb8bb2cb..d101a6d2522 100644 --- a/lib/gitlab/x509/signature.rb +++ b/lib/gitlab/x509/signature.rb @@ -110,8 +110,6 @@ module Gitlab else false end - else - nil end rescue StandardError nil |