diff options
Diffstat (limited to 'lib/gitlab')
324 files changed, 18495 insertions, 1510 deletions
diff --git a/lib/gitlab/abuse.rb b/lib/gitlab/abuse.rb index cc95d3c1e0c..7db99d4b037 100644 --- a/lib/gitlab/abuse.rb +++ b/lib/gitlab/abuse.rb @@ -3,10 +3,10 @@ module Gitlab module Abuse CONFIDENCE_LEVELS = { - certain: 1.0, - likely: 0.8, + certain: 1.0, + likely: 0.8, uncertain: 0.5, - unknown: 0.0 + unknown: 0.0 }.freeze class << self diff --git a/lib/gitlab/access.rb b/lib/gitlab/access.rb index 3e09d488bc3..fa025a2658f 100644 --- a/lib/gitlab/access.rb +++ b/lib/gitlab/access.rb @@ -41,9 +41,9 @@ module Gitlab def options { - "Guest" => GUEST, - "Reporter" => REPORTER, - "Developer" => DEVELOPER, + "Guest" => GUEST, + "Reporter" => REPORTER, + "Developer" => DEVELOPER, "Maintainer" => MAINTAINER } end @@ -62,9 +62,9 @@ module Gitlab def sym_options { - guest: GUEST, - reporter: REPORTER, - developer: DEVELOPER, + guest: GUEST, + reporter: REPORTER, + developer: DEVELOPER, maintainer: MAINTAINER } end @@ -120,9 +120,9 @@ module Gitlab def project_creation_string_options { - 'noone' => NO_ONE_PROJECT_ACCESS, - 'maintainer' => MAINTAINER_PROJECT_ACCESS, - 'developer' => DEVELOPER_MAINTAINER_PROJECT_ACCESS + 'noone' => NO_ONE_PROJECT_ACCESS, + 'maintainer' => MAINTAINER_PROJECT_ACCESS, + 'developer' => DEVELOPER_MAINTAINER_PROJECT_ACCESS } end @@ -147,7 +147,7 @@ module Gitlab def subgroup_creation_string_options { - 'owner' => OWNER_SUBGROUP_ACCESS, + 'owner' => OWNER_SUBGROUP_ACCESS, 'maintainer' => MAINTAINER_SUBGROUP_ACCESS } end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 2d769148c5f..01dcb95eab5 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -149,6 +149,10 @@ module Gitlab severity_mapping.fetch(severity_raw.to_s.downcase, UNMAPPED_SEVERITY) end + def source + monitoring_tool || integration&.name + end + private def plain_gitlab_fingerprint diff --git a/lib/gitlab/alert_management/payload/generic.rb b/lib/gitlab/alert_management/payload/generic.rb index 15238b5e50f..18e65779ead 100644 --- a/lib/gitlab/alert_management/payload/generic.rb +++ b/lib/gitlab/alert_management/payload/generic.rb @@ -6,6 +6,7 @@ module Gitlab module Payload class Generic < Base DEFAULT_TITLE = 'New: Alert' + DEFAULT_SOURCE = 'Generic Alert Endpoint' attribute :description, paths: 'description' attribute :ends_at, paths: 'end_time', type: :time @@ -22,6 +23,14 @@ module Gitlab attribute :plain_gitlab_fingerprint, paths: 'fingerprint' private :plain_gitlab_fingerprint + + def resolved? + ends_at.present? + end + + def source + super || DEFAULT_SOURCE + end end end end diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index d0d8d68362e..ac9c465bf7d 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -105,9 +105,8 @@ module Gitlab private def use_aggregated_backend? - group.present? && # for now it's only available on the group-level - aggregation.enabled && - Feature.enabled?(:use_vsa_aggregated_tables, group) + # for now it's only available on the group-level + group.present? && aggregation.enabled end def aggregation_attributes diff --git a/lib/gitlab/analytics/date_filler.rb b/lib/gitlab/analytics/date_filler.rb new file mode 100644 index 00000000000..aa3db9f3635 --- /dev/null +++ b/lib/gitlab/analytics/date_filler.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +module Gitlab + module Analytics + # This class generates a date => value hash without gaps in the data points. + # + # Simple usage: + # + # > # We have the following data for the last 5 day: + # > input = { 3.days.ago.to_date => 10, Date.today => 5 } + # + # > # Format this data, so we can chart the complete date range: + # > Gitlab::Analytics::DateFiller.new(input, from: 4.days.ago, to: Date.today, default_value: 0).fill + # > { + # > Sun, 28 Aug 2022=>0, + # > Mon, 29 Aug 2022=>10, + # > Tue, 30 Aug 2022=>0, + # > Wed, 31 Aug 2022=>0, + # > Thu, 01 Sep 2022=>5 + # > } + # + # Parameters: + # + # **input** + # A Hash containing data for the series or the chart. The key is a Date object + # or an object which can be converted to Date. + # + # **from** + # Start date of the range + # + # **to** + # End date of the range + # + # **period** + # Specifies the period in wich the dates should be generated. Options: + # + # - :day, generate date-value pair for each day in the given period + # - :week, generate date-value pair for each week (beginning of the week date) + # - :month, generate date-value pair for each week (beginning of the month date) + # + # Note: the Date objects in the `input` should follow the same pattern (beginning of ...) + # + # **default_value** + # + # Which value use when the `input` Hash does not contain data for the given day. + # + # **date_formatter** + # + # How to format the dates in the resulting hash. + class DateFiller + DEFAULT_DATE_FORMATTER = -> (date) { date } + PERIOD_STEPS = { + day: 1.day, + week: 1.week, + month: 1.month + }.freeze + + def initialize( + input, + from:, + to:, + period: :day, + default_value: nil, + date_formatter: DEFAULT_DATE_FORMATTER) + @input = input.transform_keys(&:to_date) + @from = from.to_date + @to = to.to_date + @period = period + @default_value = default_value + @date_formatter = date_formatter + end + + def fill + data = {} + + current_date = from + loop do + transformed_date = transform_date(current_date) + break if transformed_date > to + + formatted_date = date_formatter.call(transformed_date) + + value = input.delete(transformed_date) + data[formatted_date] = value.nil? ? default_value : value + + current_date = (current_date + PERIOD_STEPS.fetch(period)).to_date + end + + raise "Input contains values which doesn't fall under the given period!" if input.any? + + data + end + + private + + attr_reader :input, :from, :to, :period, :default_value, :date_formatter + + def transform_date(date) + case period + when :day + date.beginning_of_day.to_date + when :week + date.beginning_of_week.to_date + when :month + date.beginning_of_month.to_date + else + raise "Unknown period given: #{period}" + end + end + end + end +end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index a2d79b189a3..507f94d87a5 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -16,40 +16,43 @@ module Gitlab # and only do that when it's needed. def rate_limits # rubocop:disable Metrics/AbcSize { - issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, - notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, - project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, - project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, + issues_create: { threshold: -> { application_settings.issues_create_limit }, interval: 1.minute }, + notes_create: { threshold: -> { application_settings.notes_create_limit }, interval: 1.minute }, + project_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, + project_download_export: { threshold: -> { application_settings.project_download_export_limit }, interval: 1.minute }, project_repositories_archive: { threshold: 5, interval: 1.minute }, - project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, - project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, - project_testing_hook: { threshold: 5, interval: 1.minute }, - play_pipeline_schedule: { threshold: 1, interval: 1.minute }, - raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, - group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, - group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, - group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, - group_testing_hook: { threshold: 5, interval: 1.minute }, - profile_add_new_email: { threshold: 5, interval: 1.minute }, - web_hook_calls: { interval: 1.minute }, - web_hook_calls_mid: { interval: 1.minute }, - web_hook_calls_low: { interval: 1.minute }, - users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, - username_exists: { threshold: 20, interval: 1.minute }, - user_sign_up: { threshold: 20, interval: 1.minute }, - user_sign_in: { threshold: 5, interval: 10.minutes }, - profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, - profile_update_username: { threshold: 10, interval: 1.minute }, - update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, - auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, - search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, - search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, - gitlab_shell_operation: { threshold: 600, interval: 1.minute }, - pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, - temporary_email_failure: { threshold: 50, interval: 1.day }, - project_testing_integration: { threshold: 5, interval: 1.minute }, - email_verification: { threshold: 10, interval: 10.minutes }, - email_verification_code_send: { threshold: 10, interval: 1.hour } + project_generate_new_export: { threshold: -> { application_settings.project_export_limit }, interval: 1.minute }, + project_import: { threshold: -> { application_settings.project_import_limit }, interval: 1.minute }, + project_testing_hook: { threshold: 5, interval: 1.minute }, + play_pipeline_schedule: { threshold: 1, interval: 1.minute }, + raw_blob: { threshold: -> { application_settings.raw_blob_request_limit }, interval: 1.minute }, + group_export: { threshold: -> { application_settings.group_export_limit }, interval: 1.minute }, + group_download_export: { threshold: -> { application_settings.group_download_export_limit }, interval: 1.minute }, + group_import: { threshold: -> { application_settings.group_import_limit }, interval: 1.minute }, + group_testing_hook: { threshold: 5, interval: 1.minute }, + profile_add_new_email: { threshold: 5, interval: 1.minute }, + web_hook_calls: { interval: 1.minute }, + web_hook_calls_mid: { interval: 1.minute }, + web_hook_calls_low: { interval: 1.minute }, + users_get_by_id: { threshold: -> { application_settings.users_get_by_id_limit }, interval: 10.minutes }, + username_exists: { threshold: 20, interval: 1.minute }, + user_sign_up: { threshold: 20, interval: 1.minute }, + user_sign_in: { threshold: 5, interval: 10.minutes }, + profile_resend_email_confirmation: { threshold: 5, interval: 1.minute }, + profile_update_username: { threshold: 10, interval: 1.minute }, + update_environment_canary_ingress: { threshold: 1, interval: 1.minute }, + auto_rollback_deployment: { threshold: 1, interval: 3.minutes }, + search_rate_limit: { threshold: -> { application_settings.search_rate_limit }, interval: 1.minute }, + search_rate_limit_unauthenticated: { threshold: -> { application_settings.search_rate_limit_unauthenticated }, interval: 1.minute }, + gitlab_shell_operation: { threshold: 600, interval: 1.minute }, + pipelines_create: { threshold: -> { application_settings.pipeline_limit_per_project_user_sha }, interval: 1.minute }, + temporary_email_failure: { threshold: 300, interval: 1.day }, + permanent_email_failure: { threshold: 5, interval: 1.day }, + project_testing_integration: { threshold: 5, interval: 1.minute }, + email_verification: { threshold: 10, interval: 10.minutes }, + email_verification_code_send: { threshold: 10, interval: 1.hour }, + namespace_exists: { threshold: 20, interval: 1.minute }, + fetch_google_ip_list: { threshold: 10, interval: 1.minute } }.freeze end @@ -130,16 +133,16 @@ module Gitlab # @param logger [Logger] Logger to log request to a specific log file. Defaults to Gitlab::AuthLogger def log_request(request, type, current_user, logger = Gitlab::AuthLogger) request_information = { - message: 'Application_Rate_Limiter_Request', - env: type, - remote_ip: request.ip, + message: 'Application_Rate_Limiter_Request', + env: type, + remote_ip: request.ip, request_method: request.request_method, - path: request.fullpath + path: request.fullpath } if current_user request_information.merge!({ - user_id: current_user.id, + user_id: current_user.id, username: current_user.username }) end diff --git a/lib/gitlab/application_rate_limiter/increment_per_action.rb b/lib/gitlab/application_rate_limiter/increment_per_action.rb index c99d03f1344..a3343c8a97c 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_action.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_action.rb @@ -5,9 +5,9 @@ module Gitlab class IncrementPerAction < BaseStrategy def increment(cache_key, expiry) with_redis do |redis| - redis.pipelined do - redis.incr(cache_key) - redis.expire(cache_key, expiry) + redis.pipelined do |pipeline| + pipeline.incr(cache_key) + pipeline.expire(cache_key, expiry) end.first end end diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb index 8b4197cfff9..7a68dd104a8 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -9,10 +9,10 @@ module Gitlab def increment(cache_key, expiry) with_redis do |redis| - redis.pipelined do - redis.sadd(cache_key, resource_key) - redis.expire(cache_key, expiry) - redis.scard(cache_key) + redis.pipelined do |pipeline| + pipeline.sadd(cache_key, resource_key) + pipeline.expire(cache_key, expiry) + pipeline.scard(cache_key) end.last end end diff --git a/lib/gitlab/audit/auditor.rb b/lib/gitlab/audit/auditor.rb index c96be19f02d..4a6e4e2e06e 100644 --- a/lib/gitlab/audit/auditor.rb +++ b/lib/gitlab/audit/auditor.rb @@ -117,7 +117,7 @@ module Gitlab # Only capture real users for successful authentication events. user: author_if_user, user_name: @author.name, - ip_address: @ip_address, + ip_address: Gitlab::RequestContext.instance.client_ip || @author.current_sign_in_ip, result: AuthenticationEvent.results[:success], provider: @authentication_provider } diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb new file mode 100644 index 00000000000..af5dc9f4b44 --- /dev/null +++ b/lib/gitlab/audit/type/definition.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module Gitlab + module Audit + module Type + class Definition + include ActiveModel::Validations + + attr_reader :path + attr_reader :attributes + + validate :validate_schema + validate :validate_file_name + + InvalidAuditEventTypeError = Class.new(StandardError) + + AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') + AUDIT_EVENT_TYPE_SCHEMA = JSONSchemer.schema(AUDIT_EVENT_TYPE_SCHEMA_PATH) + + # The PARAMS in config/audit_events/types/type_schema.json + PARAMS = %i[ + name + description + introduced_by_issue + introduced_by_mr + group + milestone + saved_to_database + streamed + ].freeze + + PARAMS.each do |param| + define_method(param) do + attributes[param] + end + end + + def initialize(path, opts = {}) + @path = path + @attributes = {} + + # assign nil, for all unknown opts + PARAMS.each do |param| + @attributes[param] = opts[param] + end + end + + def key + name.to_sym + end + + private + + def validate_schema + schema_errors = AUDIT_EVENT_TYPE_SCHEMA + .validate(attributes.to_h.deep_stringify_keys) + .map { |error| JSONSchemer::Errors.pretty(error) } + + errors.add(:base, schema_errors) if schema_errors.present? + end + + def validate_file_name + # ignoring Style/GuardClause because if we move this into one line, we cause Layout/LineLength errors + # rubocop:disable Style/GuardClause + unless File.basename(path, ".yml") == name + errors.add(:base, "Audit event type '#{name}' has an invalid path: '#{path}'. " \ + "'#{name}' must match the filename") + end + # rubocop:enable Style/GuardClause + end + + class << self + def paths + @paths ||= [Rails.root.join('config', 'audit_events', 'types', '*.yml')] + end + + def definitions + # We lazily load all definitions + @definitions ||= load_all! + end + + def get(key) + definitions[key.to_sym] + end + + private + + def load_all! + paths.each_with_object({}) do |glob_path, definitions| + load_all_from_path!(definitions, glob_path) + end + end + + def load_all_from_path!(definitions, glob_path) + Dir.glob(glob_path).each do |path| + definition = load_from_file(path) + + if previous = definitions[definition.key] + raise InvalidAuditEventTypeError, "Audit event type '#{definition.key}' " \ + "is already defined in '#{previous.path}'" + end + + definitions[definition.key] = definition + end + end + + def load_from_file(path) + definition = File.read(path) + definition = YAML.safe_load(definition) + definition.deep_symbolize_keys! + + new(path, definition).tap(&:validate!) + rescue StandardError => e + raise InvalidAuditEventTypeError, "Invalid definition for `#{path}`: #{e.message}" + end + end + end + end + end +end + +Gitlab::Audit::Type::Definition.prepend_mod diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 82c6411c712..9dafd59561a 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -7,8 +7,8 @@ module Gitlab class Config NET_LDAP_ENCRYPTION_METHOD = { simple_tls: :simple_tls, - start_tls: :start_tls, - plain: nil + start_tls: :start_tls, + plain: nil }.freeze attr_accessor :provider, :options @@ -193,11 +193,11 @@ module Gitlab def default_attributes { - 'username' => %W(#{uid} uid sAMAccountName userid).uniq, - 'email' => %w(mail email userPrincipalName), - 'name' => 'cn', - 'first_name' => 'givenName', - 'last_name' => 'sn' + 'username' => %W(#{uid} uid sAMAccountName userid).uniq, + 'email' => %w(mail email userPrincipalName), + 'name' => 'cn', + 'first_name' => 'givenName', + 'last_name' => 'sn' } end diff --git a/lib/gitlab/auth/o_auth/auth_hash.rb b/lib/gitlab/auth/o_auth/auth_hash.rb index 37f92792d2d..82a5aad360c 100644 --- a/lib/gitlab/auth/o_auth/auth_hash.rb +++ b/lib/gitlab/auth/o_auth/auth_hash.rb @@ -33,7 +33,7 @@ module Gitlab end def password - @password ||= Gitlab::Utils.force_utf8(::User.random_password.downcase) + @password ||= Gitlab::Utils.force_utf8(::User.random_password) end def location @@ -103,7 +103,7 @@ module Gitlab { username: username, - email: email + email: email } end end diff --git a/lib/gitlab/auth/o_auth/provider.rb b/lib/gitlab/auth/o_auth/provider.rb index 1a25ed10d81..2ce8677c8b7 100644 --- a/lib/gitlab/auth/o_auth/provider.rb +++ b/lib/gitlab/auth/o_auth/provider.rb @@ -5,14 +5,14 @@ module Gitlab module OAuth class Provider LABELS = { - "alicloud" => "AliCloud", - "dingtalk" => "DingTalk", - "github" => "GitHub", - "gitlab" => "GitLab.com", - "google_oauth2" => "Google", - "azure_oauth2" => "Azure AD", + "alicloud" => "AliCloud", + "dingtalk" => "DingTalk", + "github" => "GitHub", + "gitlab" => "GitLab.com", + "google_oauth2" => "Google", + "azure_oauth2" => "Azure AD", "azure_activedirectory_v2" => "Azure AD v2", - 'atlassian_oauth2' => 'Atlassian' + 'atlassian_oauth2' => 'Atlassian' }.freeze def self.authentication(user, provider) @@ -68,7 +68,9 @@ module Gitlab nil end else - provider = Gitlab.config.omniauth.providers.find { |provider| provider.name == name } + provider = Gitlab.config.omniauth.providers.find do |provider| + provider.name == name || (provider.name == 'openid_connect' && provider.args.name == name) + end merge_provider_args_with_defaults!(provider) provider diff --git a/lib/gitlab/auth/o_auth/user.rb b/lib/gitlab/auth/o_auth/user.rb index 7d9c4c0d7c1..1fed2b263da 100644 --- a/lib/gitlab/auth/o_auth/user.rb +++ b/lib/gitlab/auth/o_auth/user.rb @@ -240,11 +240,11 @@ module Gitlab valid_username = Uniquify.new.string(valid_username) { |s| !NamespacePathValidator.valid_path?(s) } { - name: name.strip.presence || valid_username, - username: valid_username, - email: email, - password: auth_hash.password, - password_confirmation: auth_hash.password, + name: name.strip.presence || valid_username, + username: valid_username, + email: email, + password: auth_hash.password, + password_confirmation: auth_hash.password, password_automatically_set: true } end diff --git a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb index 9cf1b2247a7..88ad48c3db7 100644 --- a/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb +++ b/lib/gitlab/auth/otp/strategies/forti_authenticator/manual_otp.rb @@ -34,7 +34,7 @@ module Gitlab end def body - { username: user.username, + { username: user.username, token_code: @otp_code } end diff --git a/lib/gitlab/auth/user_access_denied_reason.rb b/lib/gitlab/auth/user_access_denied_reason.rb index ff6dc7313bb..322dfa74d09 100644 --- a/lib/gitlab/auth/user_access_denied_reason.rb +++ b/lib/gitlab/auth/user_access_denied_reason.rb @@ -57,3 +57,5 @@ module Gitlab end end end + +Gitlab::Auth::UserAccessDeniedReason.prepend_mod diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb new file mode 100644 index 00000000000..2ee0594d0a6 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Backfills the `vulnerability_reads.casted_cluster_agent_id` column + class BackfillClusterAgentsHasVulnerabilities < Gitlab::BackgroundMigration::BatchedMigrationJob + VULNERABILITY_READS_JOIN = <<~SQL + INNER JOIN vulnerability_reads + ON vulnerability_reads.casted_cluster_agent_id = cluster_agents.id AND + vulnerability_reads.project_id = cluster_agents.project_id AND + vulnerability_reads.report_type = 7 + SQL + + RELATION = ->(relation) do + relation + .where(has_vulnerabilities: false) + end + + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: RELATION + ) do |sub_batch| + sub_batch + .joins(VULNERABILITY_READS_JOIN) + .update_all(has_vulnerabilities: true) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb index b9151343d6a..2d64b7378be 100644 --- a/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb +++ b/lib/gitlab/background_migration/backfill_draft_status_on_merge_requests_with_corrected_regex.rb @@ -9,6 +9,7 @@ module Gitlab # Migration only version of MergeRequest table class MergeRequest < ::ApplicationRecord include EachBatch + validates :suggested_reviewers, json_schema: { filename: 'merge_request_suggested_reviewers' } CORRECTED_REGEXP_STR = "^(\\[draft\\]|\\(draft\\)|draft:|draft|\\[WIP\\]|WIP:|WIP)" diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index 814f5a897a9..ce4c4a28b37 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -22,7 +22,7 @@ module Gitlab ProjectFeature.connection.execute( <<~SQL UPDATE project_features pf - SET package_registry_access_level = (CASE p.packages_enabled + SET package_registry_access_level = (CASE p.packages_enabled WHEN true THEN (CASE p.visibility_level WHEN #{PROJECT_PUBLIC} THEN #{FEATURE_PUBLIC} WHEN #{PROJECT_INTERNAL} THEN #{FEATURE_ENABLED} diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb new file mode 100644 index 00000000000..815c346bb39 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id + class BackfillProjectNamespaceOnIssues < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { + relation.joins("INNER JOIN projects ON projects.id = issues.project_id") + .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil }) + } + ) do |sub_batch| + connection.execute <<~SQL + UPDATE issues + SET namespace_id = projects.project_namespace_id + FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id) + WHERE issues.id = issue_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_repositories.rb b/lib/gitlab/background_migration/backfill_project_repositories.rb index 05e2ed72fb3..c49ef9d10f5 100644 --- a/lib/gitlab/background_migration/backfill_project_repositories.rb +++ b/lib/gitlab/background_migration/backfill_project_repositories.rb @@ -212,8 +212,8 @@ module Gitlab def build_attributes_for_project(project) { project_id: project.id, - shard_id: find_shard_id(project.repository_storage), - disk_path: project.disk_path + shard_id: find_shard_id(project.repository_storage), + disk_path: project.disk_path } end diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 728b60f7a0e..0c41d6af209 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -10,16 +10,12 @@ module Gitlab vulnerability_reads.project_id = cluster_agents.project_id SQL - RELATION = ->(relation) do - relation - .where(report_type: 7) - end + CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 + + scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) } def perform - each_sub_batch( - operation_name: :update_all, - batching_scope: RELATION - ) do |sub_batch| + each_sub_batch(operation_name: :update_all) do |sub_batch| sub_batch .joins(CLUSTER_AGENTS_JOIN) .update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)') diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index 32962f2bb89..86d53ad798d 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -4,11 +4,9 @@ module Gitlab module BackgroundMigration # Backfills the `issues.work_item_type_id` column, replacing any # instances of `NULL` with the appropriate `work_item_types.id` based on `issues.issue_type` - class BackfillWorkItemTypeIdForIssues + class BackfillWorkItemTypeIdForIssues < BatchedMigrationJob # Basic AR model for issues table class MigrationIssue < ApplicationRecord - include ::EachBatch - self.table_name = 'issues' scope :base_query, ->(base_type) { where(work_item_type_id: nil, issue_type: base_type) } @@ -16,29 +14,27 @@ module Gitlab MAX_UPDATE_RETRIES = 3 - def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms, base_type, base_type_id) - parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id, base_type) + scope_to ->(relation) { + relation.where(issue_type: base_type) + } + + job_arguments :base_type, :base_type_id - parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch| + def perform + each_sub_batch( + operation_name: :update_all, + batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } + ) do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) # The query need to be reconstructed because .each_batch modifies the default scope # See: https://gitlab.com/gitlab-org/gitlab/-/issues/330510 reconstructed_sub_batch = MigrationIssue.unscoped.base_query(base_type).where(id: first..last) - batch_metrics.time_operation(:update_all) do - update_with_retry(reconstructed_sub_batch, base_type_id) - end - - pause_ms = 0 if pause_ms < 0 - sleep(pause_ms * 0.001) + update_with_retry(reconstructed_sub_batch, base_type_id) end end - def batch_metrics - @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new - end - private # Retry mechanism required as update statements on the issues table will randomly take longer than @@ -64,10 +60,6 @@ module Gitlab def update_batch(sub_batch, base_type_id) sub_batch.update_all(work_item_type_id: base_type_id) end - - def relation_scoped_to_range(source_table, source_key_column, start_id, end_id, base_type) - MigrationIssue.where(source_key_column => start_id..end_id).base_query(base_type) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb deleted file mode 100644 index 7d5fef67c25..00000000000 --- a/lib/gitlab/background_migration/batching_strategies/backfill_issue_work_item_type_batching_strategy.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - module BatchingStrategies - # Batching class to use for back-filling issue's work_item_type_id for a single issue type. - # Batches will be scoped to records where the foreign key is NULL and only of a given issue type - # - # If no more batches exist in the table, returns nil. - class BackfillIssueWorkItemTypeBatchingStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments:, job_class: nil) - issue_type = job_arguments.first - - relation.where(issue_type: issue_type) - end - end - end - end -end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb index 9ad119310f7..72da2b5a2b7 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_project_statistics_with_container_registry_size_batching_strategy.rb @@ -3,18 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling project_statistic's container_registry_size. - # Batches will be scoped to records where the project_ids are migrated - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillProjectStatisticsWithContainerRegistrySizeBatchingStrategy < PrimaryKeyBatchingStrategy - MIGRATION_PHASE_1_ENDED_AT = Date.new(2022, 01, 23).freeze - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(created_at: MIGRATION_PHASE_1_ENDED_AT..).or( - relation.where(migration_state: 'import_done') - ).select(:project_id).distinct - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb index f0d015198dc..c2fa00f66de 100644 --- a/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/backfill_vulnerability_reads_cluster_agent_batching_strategy.rb @@ -3,16 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for back-filling vulnerability_read's casted_cluster_agent_id from cluster_agent_id. - # Batches will be scoped to records where the report_type belongs to cluster_image_scanning. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class BackfillVulnerabilityReadsClusterAgentBatchingStrategy < PrimaryKeyBatchingStrategy - CLUSTER_IMAGE_SCANNING_REPORT_TYPE = 7 - - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb index e1855b6cfee..9504d4eec11 100644 --- a/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/dismissed_vulnerabilities_strategy.rb @@ -3,14 +3,9 @@ module Gitlab module BackgroundMigration module BatchingStrategies - # Batching class to use for setting state in vulnerabilitites table. - # Batches will be scoped to records where the dismissed_at is set. - # - # If no more batches exist in the table, returns nil. + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/93771 class DismissedVulnerabilitiesStrategy < PrimaryKeyBatchingStrategy - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation.where.not(dismissed_at: nil) - end end end end diff --git a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb index 1ffa4a052e5..43352b1bf91 100644 --- a/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb +++ b/lib/gitlab/background_migration/batching_strategies/primary_key_batching_strategy.rb @@ -22,8 +22,8 @@ module Gitlab def next_batch(table_name, column_name, batch_min_value:, batch_size:, job_arguments:, job_class: nil) model_class = define_batchable_model(table_name, connection: connection) - quoted_column_name = model_class.connection.quote_column_name(column_name) - relation = model_class.where("#{quoted_column_name} >= ?", batch_min_value) + arel_column = model_class.arel_table[column_name] + relation = model_class.where(arel_column.gteq(batch_min_value)) if job_class relation = filter_batch(relation, @@ -32,11 +32,10 @@ module Gitlab ) end - relation = apply_additional_filters(relation, job_arguments: job_arguments, job_class: job_class) next_batch_bounds = nil relation.each_batch(of: batch_size, column: column_name) do |batch| # rubocop:disable Lint/UnreachableLoop - next_batch_bounds = batch.pick(Arel.sql("MIN(#{quoted_column_name}), MAX(#{quoted_column_name})")) + next_batch_bounds = batch.pick(arel_column.minimum, arel_column.maximum) break end @@ -44,15 +43,6 @@ module Gitlab next_batch_bounds end - # Deprecated - # - # Use `scope_to` to define additional filters on the migration job class. - # - # see https://docs.gitlab.com/ee/development/database/batched_background_migrations.html#adding-additional-filters. - def apply_additional_filters(relation, job_arguments: [], job_class: nil) - relation - end - private def filter_batch(relation, table_name:, column_name:, job_class:, job_arguments: []) diff --git a/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb new file mode 100644 index 00000000000..49525479637 --- /dev/null +++ b/lib/gitlab/background_migration/batching_strategies/remove_backfilled_job_artifacts_expire_at_batching_strategy.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module BatchingStrategies + # Used to apply additional filters to the batching table, migrated to + # use BatchedMigrationJob#filter_batch with https://gitlab.com/gitlab-org/gitlab/-/merge_requests/96478 + class RemoveBackfilledJobArtifactsExpireAtBatchingStrategy < PrimaryKeyBatchingStrategy + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb new file mode 100644 index 00000000000..739197898d9 --- /dev/null +++ b/lib/gitlab/background_migration/delete_approval_rules_with_vulnerability.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This class doesn't delete approval rules + # as this feature exists only in EE + class DeleteApprovalRulesWithVulnerability < BatchedMigrationJob + def perform + end + end + end +end + +# rubocop:disable Layout/LineLength +Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability.prepend_mod_with('Gitlab::BackgroundMigration::DeleteApprovalRulesWithVulnerability') +# rubocop:enable Layout/LineLength diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb new file mode 100644 index 00000000000..35ac42f76ab --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidGroupMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) do + relation.where(source_type: 'Namespace') + .joins('LEFT OUTER JOIN namespaces ON members.source_id = namespaces.id') + .where(namespaces: { id: nil }) + end + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_ids = sub_batch.map(&:id) + Gitlab::AppLogger.info({ message: 'Removing invalid group member records', + deleted_count: invalid_ids.size, ids: invalid_ids }) + + sub_batch.delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb new file mode 100644 index 00000000000..3c60f765c29 --- /dev/null +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation + scope_to ->(relation) { relation.where(source_type: 'Project') } + + def perform + each_sub_batch(operation_name: :delete_all) do |sub_batch| + invalid_project_members = sub_batch + .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id') + .where(projects: { id: nil }) + invalid_ids = invalid_project_members.pluck(:id) + + # the actual delete + deleted_count = invalid_project_members.delete_all + + Gitlab::AppLogger.info({ message: 'Removing invalid project member records', + deleted_count: deleted_count, + ids: invalid_ids }) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb new file mode 100644 index 00000000000..824054b31f2 --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for public projects created after 17/02/2022 + class DisableLegacyOpenSourceLicenceForRecentPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob + PUBLIC = 20 + THRESHOLD_DATE = '2022-02-17 09:00:00' + + # Migration only version of `project_settings` table + class ProjectSetting < ApplicationRecord + self.table_name = 'project_settings' + end + + def perform + each_sub_batch( + operation_name: :disable_legacy_open_source_licence_for_recent_public_projects, + batching_scope: ->(relation) { + relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE) + } + ) do |sub_batch| + ProjectSetting.where(project_id: sub_batch) + .where(legacy_open_source_license_available: true) + .update_all(legacy_open_source_license_available: false) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb new file mode 100644 index 00000000000..6e4d5d8ddcb --- /dev/null +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB + class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob + scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } + + def perform + each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch| + updates = { legacy_open_source_license_available: false, updated_at: Time.current } + + sub_batch + .joins('INNER JOIN project_statistics ON project_statistics.project_id = project_settings.project_id') + .where('project_statistics.repository_size < ?', 1.megabyte) + .update_all(updates) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb index 3605b157f4f..2bf631c6c7d 100644 --- a/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb +++ b/lib/gitlab/background_migration/mailers/unconfirm_mailer.rb @@ -11,7 +11,7 @@ module Gitlab @user = user @verification_from_mail = Gitlab.config.gitlab.email_from - mail( + mail_with_locale( template_path: 'unconfirm_mailer', template_name: 'unconfirm_notification_email', to: @user.notification_email_or_default, diff --git a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb index 72380af2c53..9a42d035285 100644 --- a/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb +++ b/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb @@ -58,7 +58,7 @@ class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid # r development: "a143e9e2-41b3-47bc-9a19-081d089229f4", test: "a143e9e2-41b3-47bc-9a19-081d089229f4", staging: "a6930898-a1b2-4365-ab18-12aa474d9b26", - production: "58dc0f06-936c-43b3-93bb-71693f1b6570" + production: "58dc0f06-936c-43b3-93bb-71693f1b6570" }.freeze NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb new file mode 100644 index 00000000000..d30263976e8 --- /dev/null +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This detects and fixes job artifacts that have `expire_at` wrongly backfilled by the migration + # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. + # These job artifacts will not be deleted and will have their `expire_at` removed. + class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + # The migration would have backfilled `expire_at` + # to midnight on the 22nd of the month of the local timezone, + # storing it as UTC time in the database. + # + # If the timezone setting has changed since the migration, + # the `expire_at` stored in the database could have changed to a different local time other than midnight. + # For example: + # - changing timezone from UTC+02:00 to UTC+02:30 would change the `expire_at` in local time 00:00:00 to 00:30:00. + # - changing timezone from UTC+00:00 to UTC-01:00 would change the `expire_at` in local time 00:00:00 to 23:00:00 + # on the previous day (21st). + # + # Therefore job artifacts that have `expire_at` exactly on the 00, 30 or 45 minute mark + # on the dates 21, 22, 23 of the month will not be deleted. + # https://en.wikipedia.org/wiki/List_of_UTC_time_offsets + EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE = <<~SQL + EXTRACT(day FROM timezone('UTC', expire_at)) IN (21, 22, 23) + AND EXTRACT(minute FROM timezone('UTC', expire_at)) IN (0, 30, 45) + AND EXTRACT(second FROM timezone('UTC', expire_at)) = 0 + SQL + + scope_to ->(relation) { + relation.where(EXPIRES_ON_21_22_23_AT_MIDNIGHT_IN_TIMEZONE) + .or(relation.where(file_type: 3)) + } + + def perform + each_sub_batch( + operation_name: :update_all + ) do |sub_batch| + sub_batch.update_all(expire_at: nil) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb new file mode 100644 index 00000000000..5b1d630bb03 --- /dev/null +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Removes obsolete wiki notes + class RemoveSelfManagedWikiNotes < BatchedMigrationJob + def perform + each_sub_batch( + operation_name: :delete_all + ) do |sub_batch| + sub_batch.where(noteable_type: 'Wiki').delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb new file mode 100644 index 00000000000..718fb0aaa71 --- /dev/null +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Renames all system notes created when an issuable task is checked/unchecked + # from `task` into `checklist item` + # `marked the task **Task 1** as incomplete` => `marked the checklist item **Task 1** as incomplete` + class RenameTaskSystemNoteToChecklistItem < BatchedMigrationJob + REPLACE_REGEX = '\Amarked\sthe\stask' + TEXT_REPLACEMENT = 'marked the checklist item' + + scope_to ->(relation) { + relation.where(system_note_metadata: { action: :task }) + } + + def perform + each_sub_batch(operation_name: :update_all) do |sub_batch| + ApplicationRecord.connection.execute <<~SQL + UPDATE notes + SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}') + FROM (#{sub_batch.select(:note_id).to_sql}) AS metadata_fields(note_id) + WHERE notes.id = note_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index fd6cbcb8d05..a0cfeed618a 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -6,11 +6,10 @@ module Gitlab class SetCorrectVulnerabilityState < BatchedMigrationJob DISMISSED_STATE = 2 + scope_to ->(relation) { relation.where.not(dismissed_at: nil) } + def perform - each_sub_batch( - operation_name: :update_vulnerabilities_state, - batching_scope: -> (relation) { relation.where.not(dismissed_at: nil) } - ) do |sub_batch| + each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch| sub_batch.update_all(state: DISMISSED_STATE) end end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 81b01395542..c8520993b8e 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,6 +3,7 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab + # rubocop:disable Rails/ApplicationController class BaseDoorkeeperController < ActionController::Base include Gitlab::Allowable include EnforcesTwoFactorAuthentication @@ -12,4 +13,5 @@ module Gitlab helper_method :can? end + # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/cache/helpers.rb b/lib/gitlab/cache/helpers.rb index 7b11d6bc9ff..48b6ca59367 100644 --- a/lib/gitlab/cache/helpers.rb +++ b/lib/gitlab/cache/helpers.rb @@ -57,9 +57,19 @@ module Gitlab # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @return [String] def cached_object(object, presenter:, presenter_args:, context:, expires_in:) - cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do - Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + misses = 0 + + json = cache.fetch(contextual_cache_key(presenter, object, context), expires_in: expires_in) do + time_action(render_type: :object) do + misses += 1 + + Gitlab::Json.dump(presenter.represent(object, **presenter_args).as_json) + end end + + increment_cache_metric(render_type: :object, total_count: 1, miss_count: misses) + + json end # Used for fetching or rendering multiple objects @@ -71,10 +81,18 @@ module Gitlab # @param expires_in [ActiveSupport::Duration, Integer] an expiry time for the cache entry # @return [Array<String>] def cached_collection(collection, presenter:, presenter_args:, context:, expires_in:) + misses = 0 + json = fetch_multi(presenter, collection, context: context, expires_in: expires_in) do |obj| - Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + time_action(render_type: :collection) do + misses += 1 + + Gitlab::Json.dump(presenter.represent(obj, **presenter_args).as_json) + end end + increment_cache_metric(render_type: :collection, total_count: collection.length, miss_count: misses) + json.values end @@ -106,6 +124,57 @@ module Gitlab contextual_cache_key(presenter, object, context) end end + + def increment_cache_metric(render_type:, total_count:, miss_count:) + return unless Feature.enabled?(:add_timing_to_certain_cache_actions) + return unless caller_id + + metric_name = :cached_object_operations_total + hit_count = total_count - miss_count + + current_transaction&.increment( + metric_name, + hit_count, + { caller_id: caller_id, render_type: render_type, cache_hit: true } + ) + + current_transaction&.increment( + metric_name, + miss_count, + { caller_id: caller_id, render_type: render_type, cache_hit: false } + ) + end + + def time_action(render_type:, &block) + if Feature.enabled?(:add_timing_to_certain_cache_actions) + real_start = Gitlab::Metrics::System.monotonic_time + + presented_object = yield + + real_duration_histogram(render_type).observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + + presented_object + else + yield + end + end + + def real_duration_histogram(render_type) + Gitlab::Metrics.histogram( + :gitlab_presentable_object_cacheless_render_real_duration_seconds, + 'Duration of generating presentable objects to be cached in real time', + { caller_id: caller_id, render_type: render_type }, + [0.1, 0.5, 1, 2] + ) + end + + def current_transaction + @current_transaction ||= ::Gitlab::Metrics::WebTransaction.current + end + + def caller_id + @caller_id ||= Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end end end end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 10233cf4228..2ab702aa4f9 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -19,11 +19,11 @@ module Gitlab }.freeze STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 }.freeze def self.convert(ansi, state = nil) diff --git a/lib/gitlab/ci/ansi2json/parser.rb b/lib/gitlab/ci/ansi2json/parser.rb index 79b42a5f5bf..fdd49df1e24 100644 --- a/lib/gitlab/ci/ansi2json/parser.rb +++ b/lib/gitlab/ci/ansi2json/parser.rb @@ -20,11 +20,11 @@ module Gitlab }.freeze STYLE_SWITCHES = { - bold: 0x01, - italic: 0x02, - underline: 0x04, - conceal: 0x08, - cross: 0x10 + bold: 0x01, + italic: 0x02, + underline: 0x04, + conceal: 0x08, + cross: 0x10 }.freeze def self.bold?(mask) diff --git a/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb b/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb deleted file mode 100644 index 690a47097c6..00000000000 --- a/lib/gitlab/ci/build/artifacts/adapters/zip_stream.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Build - module Artifacts - module Adapters - class ZipStream - MAX_DECOMPRESSED_SIZE = 100.megabytes - MAX_FILES_PROCESSED = 50 - - attr_reader :stream - - InvalidStreamError = Class.new(StandardError) - - def initialize(stream) - raise InvalidStreamError, "Stream is required" unless stream - - @stream = stream - @files_processed = 0 - end - - def each_blob - Zip::InputStream.open(stream) do |zio| - while entry = zio.get_next_entry - break if at_files_processed_limit? - next unless should_process?(entry) - - @files_processed += 1 - - yield entry.get_input_stream.read - end - end - end - - private - - def should_process?(entry) - file?(entry) && !too_large?(entry) - end - - def file?(entry) - # Check the file name as a workaround for incorrect - # file type detection when using InputStream - # https://github.com/rubyzip/rubyzip/issues/533 - entry.file? && !entry.name.end_with?('/') - end - - def too_large?(entry) - entry.size > MAX_DECOMPRESSED_SIZE - end - - def at_files_processed_limit? - @files_processed >= MAX_FILES_PROCESSED - end - end - end - end - end - end -end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 641aa71fb4e..a1a8e9288c7 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -32,7 +32,18 @@ module Gitlab end def build_attributes - attributes.merge(pipeline_attributes) + attributes.merge(pipeline_attributes, ci_stage_attributes) + end + + def ci_stage_attributes + { + ci_stage: ::Ci::Stage.new( + name: attributes[:stage], + position: attributes[:stage_idx], + pipeline: pipeline_attributes[:pipeline], + project: pipeline_attributes[:project] + ) + } end end end diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index e2b54797dc8..aebd81e7b07 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -24,7 +24,7 @@ module Gitlab private def worktree_paths(context) - return unless context.project + return [] unless context.project if @top_level_only context.top_level_worktree_paths diff --git a/lib/gitlab/ci/config/entry/current_variables.rb b/lib/gitlab/ci/config/entry/current_variables.rb new file mode 100644 index 00000000000..3b6721ec92d --- /dev/null +++ b/lib/gitlab/ci/config/entry/current_variables.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents CI/CD variables. + # The class will be renamed to `Variables` when removing the FF `ci_variables_refactoring_to_variable`. + # + class CurrentVariables < ::Gitlab::Config::Entry::ComposableHash + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: Hash + end + + # Enable these lines when removing the FF `ci_variables_refactoring_to_variable` + # and renaming this class to `Variables`. + # def self.default(**) + # {} + # end + + def value + @entries.to_h do |key, entry| + [key.to_s, entry.value] + end + end + + def value_with_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_data] + end + end + + private + + def composable_class(_name, _config) + Entry::Variable + end + + def composable_metadata + { allowed_value_data: opt(:allowed_value_data) } + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/environment.rb b/lib/gitlab/ci/config/entry/environment.rb index 96ba3553b46..a727da87308 100644 --- a/lib/gitlab/ci/config/entry/environment.rb +++ b/lib/gitlab/ci/config/entry/environment.rb @@ -54,7 +54,7 @@ module Gitlab validates :on_stop, type: String, allow_nil: true validates :kubernetes, type: Hash, allow_nil: true - validates :auto_stop_in, duration: { parser: ::Gitlab::Ci::Build::DurationParser }, allow_nil: true + validates :auto_stop_in, type: String, allow_nil: true end end diff --git a/lib/gitlab/ci/config/entry/image.rb b/lib/gitlab/ci/config/entry/image.rb index 613f7ff3370..84e31ca1fc6 100644 --- a/lib/gitlab/ci/config/entry/image.rb +++ b/lib/gitlab/ci/config/entry/image.rb @@ -11,10 +11,7 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable validations do - validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? + validates :config, allowed_keys: IMAGEABLE_ALLOWED_KEYS end def value @@ -25,7 +22,7 @@ module Gitlab name: @config[:name], entrypoint: @config[:entrypoint], ports: (ports_value if ports_defined?), - pull_policy: (ci_docker_image_pull_policy_enabled? ? pull_policy_value : nil) + pull_policy: pull_policy_value }.compact else {} diff --git a/lib/gitlab/ci/config/entry/imageable.rb b/lib/gitlab/ci/config/entry/imageable.rb index f045ee3d549..1aecfee9ab9 100644 --- a/lib/gitlab/ci/config/entry/imageable.rb +++ b/lib/gitlab/ci/config/entry/imageable.rb @@ -13,7 +13,6 @@ module Gitlab include ::Gitlab::Config::Entry::Configurable IMAGEABLE_ALLOWED_KEYS = %i[name entrypoint ports pull_policy].freeze - IMAGEABLE_LEGACY_ALLOWED_KEYS = %i[name entrypoint ports].freeze included do include ::Gitlab::Config::Entry::Validatable @@ -47,10 +46,6 @@ module Gitlab opt(:with_image_ports) end - def ci_docker_image_pull_policy_enabled? - ::Feature.enabled?(:ci_docker_image_pull_policy) - end - def skip_config_hash_validation? true end diff --git a/lib/gitlab/ci/config/entry/legacy_variables.rb b/lib/gitlab/ci/config/entry/legacy_variables.rb new file mode 100644 index 00000000000..5379f707537 --- /dev/null +++ b/lib/gitlab/ci/config/entry/legacy_variables.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents environment variables. + # This is legacy implementation and will be removed with the FF `ci_variables_refactoring_to_variable`. + # + class LegacyVariables < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + ALLOWED_VALUE_DATA = %i[value description].freeze + + validations do + validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? + validates :config, variables: true, unless: :use_value_data? + end + + def value + @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } + end + + def value_with_data + @config.to_h { |key, value| [key.to_s, expand_value(value)] } + end + + def use_value_data? + opt(:use_value_data) + end + + private + + def expand_value(value) + if value.is_a?(Hash) + { value: value[:value].to_s, description: value[:description] }.compact + else + { value: value.to_s } + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 78794f524f4..2d2032b1d8c 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -29,7 +29,7 @@ module Gitlab in: %i[only except start_in], message: 'key may not be used with `rules`' }, - if: :has_rules? + if: :has_rules? with_options allow_nil: true do validates :extends, array_of_strings_or_string: true @@ -120,7 +120,7 @@ module Gitlab stage: stage_value, extends: extends, rules: rules_value, - job_variables: variables_value.to_h, + job_variables: variables_entry.value_with_data, root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index ff11c757dfa..57e89bd7bc5 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -48,9 +48,10 @@ module Gitlab description: 'Script that will be executed after each job.', reserved: true + # use_value_data will be removed with the FF ci_variables_refactoring_to_variable entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { use_value_data: true }, + metadata: { use_value_data: true, allowed_value_data: %i[value description] }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/service.rb b/lib/gitlab/ci/config/entry/service.rb index 0e19447dff8..4b3a9990df4 100644 --- a/lib/gitlab/ci/config/entry/service.rb +++ b/lib/gitlab/ci/config/entry/service.rb @@ -11,14 +11,9 @@ module Gitlab include ::Gitlab::Ci::Config::Entry::Imageable ALLOWED_KEYS = %i[command alias variables].freeze - LEGACY_ALLOWED_KEYS = %i[command alias variables].freeze validations do - validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS, - if: :ci_docker_image_pull_policy_enabled? - validates :config, allowed_keys: LEGACY_ALLOWED_KEYS + IMAGEABLE_LEGACY_ALLOWED_KEYS, - unless: :ci_docker_image_pull_policy_enabled? - + validates :config, allowed_keys: ALLOWED_KEYS + IMAGEABLE_ALLOWED_KEYS validates :command, array_of_strings: true, allow_nil: true validates :alias, type: String, allow_nil: true validates :alias, type: String, presence: true, unless: ->(record) { record.ports.blank? } @@ -43,7 +38,7 @@ module Gitlab { name: @config } elsif hash? @config.merge( - pull_policy: (pull_policy_value if ci_docker_image_pull_policy_enabled?) + pull_policy: pull_policy_value ).compact else {} diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb new file mode 100644 index 00000000000..253888aadeb --- /dev/null +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents a CI/CD variable. + # + class Variable < ::Gitlab::Config::Entry::Simplifiable + strategy :SimpleVariable, if: -> (config) { SimpleVariable.applies_to?(config) } + strategy :ComplexVariable, if: -> (config) { ComplexVariable.applies_to?(config) } + + class SimpleVariable < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + Gitlab::Config::Entry::Validators::AlphanumericValidator.validate(config) + end + end + + validations do + validates :key, alphanumeric: true + validates :config, alphanumeric: true + end + + def value + @config.to_s + end + + def value_with_data + { value: @config.to_s } + end + end + + class ComplexVariable < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + class << self + def applies_to?(config) + config.is_a?(Hash) + end + end + + validations do + validates :key, alphanumeric: true + validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? + validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? + + validate do + allowed_value_data = Array(opt(:allowed_value_data)) + + if allowed_value_data.any? + extra_keys = config.keys - allowed_value_data + + errors.add(:config, "uses invalid data keys: #{extra_keys.join(', ')}") if extra_keys.present? + else + errors.add(:config, "must be a string") + end + end + end + + def value + config_value.to_s + end + + def value_with_data + { value: value, description: config_description }.compact + end + + def config_value + @config[:value] + end + + def config_description + @config[:description] + end + + def config_value_defined? + config.key?(:value) + end + + def config_description_defined? + config.key?(:description) + end + end + + class UnknownStrategy < ::Gitlab::Config::Entry::Node + def errors + ["variable definition must be either a string or a hash"] + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index efb469ee32a..0284958d9d4 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -5,43 +5,21 @@ module Gitlab class Config module Entry ## - # Entry that represents environment variables. + # Entry that represents CI/CD variables. + # CurrentVariables will be renamed to this class when removing the FF `ci_variables_refactoring_to_variable`. # - class Variables < ::Gitlab::Config::Entry::Node - include ::Gitlab::Config::Entry::Validatable - - ALLOWED_VALUE_DATA = %i[value description].freeze - - validations do - validates :config, variables: { allowed_value_data: ALLOWED_VALUE_DATA }, if: :use_value_data? - validates :config, variables: true, unless: :use_value_data? - end - - def value - @config.to_h { |key, value| [key.to_s, expand_value(value)[:value]] } + class Variables + def self.new(...) + if YamlProcessor::FeatureFlags.enabled?(:ci_variables_refactoring_to_variable) + CurrentVariables.new(...) + else + LegacyVariables.new(...) + end end def self.default(**) {} end - - def value_with_data - @config.to_h { |key, value| [key.to_s, expand_value(value)] } - end - - def use_value_data? - opt(:use_value_data) - end - - private - - def expand_value(value) - if value.is_a?(Hash) - { value: value[:value].to_s, description: value[:description] } - else - { value: value.to_s, description: nil } - end - end end end end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 278353220e4..4e01688a955 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -8,7 +8,7 @@ module Gitlab def reserved_claims super.merge( iss: Settings.gitlab.base_url, - sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", + sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", aud: Settings.gitlab.base_url ) end diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index deb20a2138c..aa594ca4049 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -6,7 +6,6 @@ module Gitlab module Sbom class Cyclonedx SUPPORTED_SPEC_VERSIONS = %w[1.4].freeze - COMPONENT_ATTRIBUTES = %w[type name version].freeze def parse!(blob, sbom_report) @report = sbom_report @@ -62,10 +61,17 @@ module Gitlab end def parse_components - data['components']&.each do |component| - next unless supported_component_type?(component['type']) + data['components']&.each do |component_data| + type = component_data['type'] + next unless supported_component_type?(type) - report.add_component(component.slice(*COMPONENT_ATTRIBUTES)) + component = ::Gitlab::Ci::Reports::Sbom::Component.new( + type: type, + name: component_data['name'], + version: component_data['version'] + ) + + report.add_component(component) end end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index ad04b3257f9..00ca723b258 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -21,11 +21,11 @@ module Gitlab def source return unless required_attributes_present? - { - 'type' => :dependency_scanning, - 'data' => data, - 'fingerprint' => fingerprint - } + ::Gitlab::Ci::Reports::Sbom::Source.new( + type: :dependency_scanning, + data: data, + fingerprint: fingerprint + ) end private diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 13a159f3745..da7faaab6ff 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -7,16 +7,16 @@ module Gitlab class Common SecurityReportParserError = Class.new(Gitlab::Ci::Parsers::ParserError) - def self.parse!(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) - new(json_data, report, vulnerability_finding_signatures_enabled, validate: validate).parse! + def self.parse!(json_data, report, signatures_enabled: false, validate: false) + new(json_data, report, signatures_enabled: signatures_enabled, validate: validate).parse! end - def initialize(json_data, report, vulnerability_finding_signatures_enabled = false, validate: false) + def initialize(json_data, report, signatures_enabled: false, validate: false) @json_data = json_data @report = report @project = report.project @validate = validate - @vulnerability_finding_signatures_enabled = vulnerability_finding_signatures_enabled + @signatures_enabled = signatures_enabled end def parse! @@ -26,7 +26,7 @@ module Gitlab raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash) - create_scanner + create_scanner(top_level_scanner_data) create_scan create_analyzer @@ -77,7 +77,7 @@ module Gitlab report_data, report.version, project: @project, - scanner: top_level_scanner + scanner: top_level_scanner_data ) end @@ -89,8 +89,8 @@ module Gitlab @report_version ||= report_data['version'] end - def top_level_scanner - @top_level_scanner ||= report_data.dig('scan', 'scanner') + def top_level_scanner_data + @top_level_scanner_data ||= report_data.dig('scan', 'scanner') end def scan_data @@ -119,7 +119,7 @@ module Gitlab evidence = create_evidence(data['evidence']) signatures = create_signatures(tracking_data(data)) - if @vulnerability_finding_signatures_enabled && !signatures.empty? + if @signatures_enabled && !signatures.empty? # NOT the signature_sha - the compare key is hashed # to create the project_fingerprint highest_priority_signature = signatures.max_by(&:priority) @@ -138,7 +138,7 @@ module Gitlab evidence: evidence, severity: parse_severity_level(data['severity']), confidence: parse_confidence_level(data['confidence']), - scanner: create_scanner(data['scanner']), + scanner: create_scanner(top_level_scanner_data || data['scanner']), scan: report&.scan, identifiers: identifiers, flags: flags, @@ -149,7 +149,7 @@ module Gitlab details: data['details'] || {}, signatures: signatures, project_id: @project.id, - vulnerability_finding_signatures_enabled: @vulnerability_finding_signatures_enabled)) + vulnerability_finding_signatures_enabled: @signatures_enabled)) end def create_signatures(tracking) @@ -208,7 +208,7 @@ module Gitlab report.analyzer = ::Gitlab::Ci::Reports::Security::Analyzer.new(**params) end - def create_scanner(scanner_data = top_level_scanner) + def create_scanner(scanner_data) return unless scanner_data.is_a?(Hash) report.add_scanner( diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index c075ada725a..28d6620e5c4 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..db4c7ab1425 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/cluster-image-scanning-report-format.json @@ -0,0 +1,977 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json new file mode 100644 index 00000000000..641cfc82e48 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/container-scanning-report-format.json @@ -0,0 +1,911 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..59aa172444d --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json new file mode 100644 index 00000000000..0e4c866794a --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dast-report-format.json @@ -0,0 +1,1287 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + }, + "discovered_at": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss.sss, representing when the vulnerability was discovered", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}\\.\\d{3}$", + "examples": [ + "2020-01-28T03:26:02.956" + ] + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json new file mode 100644 index 00000000000..652c2f48fe4 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/dependency-scanning-report-format.json @@ -0,0 +1,968 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "dependency_files", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json new file mode 100644 index 00000000000..40d4d9f5287 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json new file mode 100644 index 00000000000..cfde126dd7b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/14.1.3/secret-detection-report-format.json @@ -0,0 +1,892 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "14.1.3" + }, + "required": [ + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "format": "uri", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}\\:\\d{2}\\:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "format": "uri" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "category", + "cve", + "identifiers", + "location", + "scanner" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "category": { + "type": "string", + "minLength": 1, + "description": "Describes where this vulnerability belongs (for example, SAST, Dependency Scanning, and so on)." + }, + "name": { + "type": "string", + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "message": { + "type": "string", + "description": "A short text section that describes the vulnerability. This may include the finding's specific information." + }, + "description": { + "type": "string", + "description": "A long text section describing the vulnerability more fully." + }, + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "confidence": { + "type": "string", + "description": "How reliable the vulnerability's assessment is. Possible values are Ignore, Unknown, Experimental, Low, Medium, High, and Confirmed. Note that some analyzers may not report all these possible values.", + "enum": [ + "Ignore", + "Unknown", + "Experimental", + "Low", + "Medium", + "High", + "Confirmed" + ] + }, + "solution": { + "type": "string", + "description": "Explanation of how to fix the vulnerability." + }, + "scanner": { + "description": "Describes the scanner used to find this vulnerability.", + "type": "object", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "The scanner's ID, as a snake_case string." + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Human-readable name of the scanner." + } + } + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "format": "uri" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "format": "uri" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "cve" + ], + "properties": { + "cve": { + "type": "string", + "description": "(Deprecated - use vulnerabilities[].id instead) A fingerprint string value that represents a concrete finding. This is used to determine whether two findings are same, which may not be 100% accurate. Note that this is NOT a CVE as described by https://cve.mitre.org/." + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..7ccb39a2b8e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/cluster-image-scanning-report-format.json @@ -0,0 +1,946 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json new file mode 100644 index 00000000000..2517832853e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/container-scanning-report-format.json @@ -0,0 +1,880 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "pattern": "^[^:]+(:\\d+[^:]*)?:[^:]+(:[^:]+)?$", + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "pattern": "^[a-zA-Z0-9/_.-]+(:\\d+[a-zA-Z0-9/_.-]*)?:[a-zA-Z0-9_.-]+$", + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..a2f9eb12992 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/coverage-fuzzing-report-format.json @@ -0,0 +1,836 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json new file mode 100644 index 00000000000..10fafaf8975 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dast-report-format.json @@ -0,0 +1,1241 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json new file mode 100644 index 00000000000..ade1ce9ea8f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/dependency-scanning-report-format.json @@ -0,0 +1,944 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json new file mode 100644 index 00000000000..9fae45d728e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/sast-report-format.json @@ -0,0 +1,831 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json new file mode 100644 index 00000000000..fca00e17f26 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.0/secret-detection-report-format.json @@ -0,0 +1,854 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.0" + }, + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/test/junit.rb b/lib/gitlab/ci/parsers/test/junit.rb index 999ffff85d2..d95ecff85cd 100644 --- a/lib/gitlab/ci/parsers/test/junit.rb +++ b/lib/gitlab/ci/parsers/test/junit.rb @@ -8,7 +8,9 @@ module Gitlab JunitParserError = Class.new(Gitlab::Ci::Parsers::ParserError) ATTACHMENT_TAG_REGEX = /\[\[ATTACHMENT\|(?<path>.+?)\]\]/.freeze - def parse!(xml_data, test_suite, job:) + def parse!(xml_data, test_report, job:) + test_suite = test_report.get_suite(job.test_suite_name) + root = Hash.from_xml(xml_data) total_parsed = 0 max_test_cases = job.max_test_cases_per_report diff --git a/lib/gitlab/ci/pipeline/chain/assign_partition.rb b/lib/gitlab/ci/pipeline/chain/assign_partition.rb new file mode 100644 index 00000000000..4b8efe13d44 --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/assign_partition.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class AssignPartition < Chain::Base + include Chain::Helpers + + def perform! + @pipeline.partition_id = find_partition_id + end + + def break? + @pipeline.errors.any? + end + + private + + def find_partition_id + if @command.creates_child_pipeline? + @command.parent_pipeline_partition_id + else + ::Ci::Pipeline.current_partition_value + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 0a6f6fd740c..14c320f77bf 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -80,6 +80,10 @@ module Gitlab bridge&.parent_pipeline end + def parent_pipeline_partition_id + parent_pipeline.partition_id if creates_child_pipeline? + end + def creates_child_pipeline? bridge&.triggers_child_pipeline? end @@ -117,8 +121,14 @@ module Gitlab end def observe_jobs_count_in_alive_pipelines + jobs_count = if Feature.enabled?(:ci_limit_active_jobs_early, project) + project.all_pipelines.jobs_count_in_alive_pipelines + else + project.all_pipelines.builds_count_in_alive_pipelines + end + metrics.active_jobs_histogram - .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines) + .observe({ plan: project.actual_plan_name }, jobs_count) end def increment_pipeline_failure_reason_counter(reason) diff --git a/lib/gitlab/ci/pipeline/chain/config/content.rb b/lib/gitlab/ci/pipeline/chain/config/content.rb index 3c150ca26bb..a14dec48619 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content.rb @@ -7,6 +7,7 @@ module Gitlab module Config class Content < Chain::Base include Chain::Helpers + include ::Gitlab::Utils::StrongMemoize SOURCES = [ Gitlab::Ci::Pipeline::Chain::Config::Content::Parameter, @@ -18,10 +19,10 @@ module Gitlab ].freeze def perform! - if config = find_config - @pipeline.build_pipeline_config(content: config.content) - @command.config_content = config.content - @pipeline.config_source = config.source + if pipeline_config&.exists? + @pipeline.build_pipeline_config(content: pipeline_config.content) + @command.config_content = pipeline_config.content + @pipeline.config_source = pipeline_config.source else error('Missing CI config file') end @@ -33,7 +34,19 @@ module Gitlab private - def find_config + def pipeline_config + strong_memoize(:pipeline_config) do + next legacy_find_config if ::Feature.disabled?(:ci_project_pipeline_config_refactoring, project) + + ::Gitlab::Ci::ProjectConfig.new( + project: project, sha: @pipeline.sha, + custom_content: @command.content, + pipeline_source: @command.source, pipeline_source_bridge: @command.bridge + ) + end + end + + def legacy_find_config sources.each do |source| config = source.new(@pipeline, @command) return config if config.exists? diff --git a/lib/gitlab/ci/pipeline/chain/config/content/source.rb b/lib/gitlab/ci/pipeline/chain/config/content/source.rb index 8bc172f93d3..69dca1568b6 100644 --- a/lib/gitlab/ci/pipeline/chain/config/content/source.rb +++ b/lib/gitlab/ci/pipeline/chain/config/content/source.rb @@ -6,6 +6,7 @@ module Gitlab module Chain module Config class Content + # When removing ci_project_pipeline_config_refactoring, this and its subclasses will be removed. class Source include Gitlab::Utils::StrongMemoize diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 245ef32f06b..3dd9b85d9b2 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -18,7 +18,9 @@ module Gitlab def ensure_environment(build) return unless build.instance_of?(::Ci::Build) && build.has_environment? - environment = ::Gitlab::Ci::Pipeline::Seed::Environment.new(build).to_resource + environment = ::Gitlab::Ci::Pipeline::Seed::Environment + .new(build, merge_request: @command.merge_request) + .to_resource if environment.persisted? build.persisted_environment = environment diff --git a/lib/gitlab/ci/pipeline/chain/validate/external.rb b/lib/gitlab/ci/pipeline/chain/validate/external.rb index 6e95c7988fc..915e48828d2 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/external.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/external.rb @@ -57,7 +57,8 @@ module Gitlab }.compact Gitlab::HTTP.post( - validation_service_url, timeout: validation_service_timeout, + validation_service_url, + timeout: validation_service_timeout, headers: headers, body: validation_service_payload.to_json ) @@ -96,13 +97,17 @@ module Gitlab last_sign_in_ip: current_user.last_sign_in_ip, sign_in_count: current_user.sign_in_count }, + credit_card: { + similar_cards_count: current_user.credit_card_validation&.similar_records&.count.to_i, + similar_holder_names_count: current_user.credit_card_validation&.similar_holder_names_count.to_i + }, pipeline: { sha: pipeline.sha, ref: pipeline.ref, type: pipeline.source }, builds: builds_validation_payload, - total_builds_count: current_user.pipelines.jobs_count_in_alive_pipelines + total_builds_count: current_user.pipelines.builds_count_in_alive_pipelines } end diff --git a/lib/gitlab/ci/pipeline/seed/build.rb b/lib/gitlab/ci/pipeline/seed/build.rb index 93106b96af2..2e4267e986b 100644 --- a/lib/gitlab/ci/pipeline/seed/build.rb +++ b/lib/gitlab/ci/pipeline/seed/build.rb @@ -148,7 +148,9 @@ module Gitlab ref: @pipeline.ref, tag: @pipeline.tag, trigger_request: @pipeline.legacy_trigger, - protected: @pipeline.protected_ref? + protected: @pipeline.protected_ref?, + partition_id: @pipeline.partition_id, + metadata_attributes: { partition_id: @pipeline.partition_id } } end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb index 6bcc71a808b..8353bc523bf 100644 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ b/lib/gitlab/ci/pipeline/seed/environment.rb @@ -5,17 +5,21 @@ module Gitlab module Pipeline module Seed class Environment < Seed::Base - attr_reader :job + attr_reader :job, :merge_request - def initialize(job) + delegate :simple_variables, to: :job + + def initialize(job, merge_request: nil) @job = job + @merge_request = merge_request end def to_resource environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| # Initialize the attributes at creation - environment.auto_stop_in = auto_stop_in + environment.auto_stop_in = expanded_auto_stop_in environment.tier = deployment_tier + environment.merge_request = merge_request end end @@ -36,6 +40,12 @@ module Gitlab def expanded_environment_name job.expanded_environment_name end + + def expanded_auto_stop_in + return unless auto_stop_in + + ExpandVariables.expand(auto_stop_in, -> { simple_variables.sort_and_expand_all }) + end end end end diff --git a/lib/gitlab/ci/pipeline/seed/stage.rb b/lib/gitlab/ci/pipeline/seed/stage.rb index 7cf6466cf4b..1c4247bd5ee 100644 --- a/lib/gitlab/ci/pipeline/seed/stage.rb +++ b/lib/gitlab/ci/pipeline/seed/stage.rb @@ -25,7 +25,8 @@ module Gitlab { name: @attributes.fetch(:name), position: @attributes.fetch(:index), pipeline: @pipeline, - project: @pipeline.project } + project: @pipeline.project, + partition_id: @pipeline.partition_id } end def seeds diff --git a/lib/gitlab/ci/processable_object_hierarchy.rb b/lib/gitlab/ci/processable_object_hierarchy.rb new file mode 100644 index 00000000000..1122361e27e --- /dev/null +++ b/lib/gitlab/ci/processable_object_hierarchy.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProcessableObjectHierarchy < ::Gitlab::ObjectHierarchy + private + + def middle_table + ::Ci::BuildNeed.arel_table + end + + def from_tables(cte) + [objects_table, cte.table, middle_table] + end + + def parent_id_column(_cte) + middle_table[:name] + end + + def ancestor_conditions(cte) + middle_table[:name].eq(objects_table[:name]).and( + middle_table[:build_id].eq(cte.table[:id]) + ) + end + + def descendant_conditions(cte) + middle_table[:build_id].eq(objects_table[:id]).and( + middle_table[:name].eq(cte.table[:name]) + ) + end + end + end +end diff --git a/lib/gitlab/ci/project_config.rb b/lib/gitlab/ci/project_config.rb new file mode 100644 index 00000000000..ded6877ef29 --- /dev/null +++ b/lib/gitlab/ci/project_config.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + # Locates project CI config + class ProjectConfig + # The order of sources is important: + # - EE uses Compliance first since it must be used first if compliance templates are enabled. + # (see ee/lib/ee/gitlab/ci/project_config.rb) + # - Parameter is used by on-demand security scanning which passes the actual CI YAML to use as argument. + # - Bridge is used for downstream pipelines since the config is defined in the bridge job. If lower in priority, + # it would evaluate the project's YAML file instead. + # - Repository / ExternalProject / Remote: their order is not important between each other. + # - AutoDevops is used as default option if nothing else is found and if AutoDevops is enabled. + SOURCES = [ + ProjectConfig::Parameter, + ProjectConfig::Bridge, + ProjectConfig::Repository, + ProjectConfig::ExternalProject, + ProjectConfig::Remote, + ProjectConfig::AutoDevops + ].freeze + + def initialize(project:, sha:, custom_content: nil, pipeline_source: nil, pipeline_source_bridge: nil) + @config = find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + end + + delegate :content, :source, to: :@config, allow_nil: true + + def exists? + !!@config&.exists? + end + + private + + def find_config(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + sources.each do |source| + config = source.new(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + return config if config.exists? + end + + nil + end + + def sources + SOURCES + end + end + end +end + +Gitlab::Ci::ProjectConfig.prepend_mod_with('Gitlab::Ci::ProjectConfig') diff --git a/lib/gitlab/ci/project_config/auto_devops.rb b/lib/gitlab/ci/project_config/auto_devops.rb new file mode 100644 index 00000000000..c6905f480a2 --- /dev/null +++ b/lib/gitlab/ci/project_config/auto_devops.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class AutoDevops < Source + def content + strong_memoize(:content) do + next unless project&.auto_devops_enabled? + + template = Gitlab::Template::GitlabCiYmlTemplate.find(template_name) + YAML.dump('include' => [{ 'template' => template.full_name }]) + end + end + + def source + :auto_devops_source + end + + private + + def template_name + 'Auto-DevOps' + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/bridge.rb b/lib/gitlab/ci/project_config/bridge.rb new file mode 100644 index 00000000000..c342ab2c215 --- /dev/null +++ b/lib/gitlab/ci/project_config/bridge.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Bridge < Source + def content + return unless pipeline_source_bridge + + pipeline_source_bridge.yaml_for_downstream + end + + def source + :bridge_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/external_project.rb b/lib/gitlab/ci/project_config/external_project.rb new file mode 100644 index 00000000000..0ed5d6fa226 --- /dev/null +++ b/lib/gitlab/ci/project_config/external_project.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class ExternalProject < Source + def content + strong_memoize(:content) do + next unless external_project_path? + + path_file, path_project, ref = extract_location_tokens + + config_location = { 'project' => path_project, 'file' => path_file } + config_location['ref'] = ref if ref.present? + + YAML.dump('include' => [config_location]) + end + end + + def source + :external_project_source + end + + private + + # Example: path/to/.gitlab-ci.yml@another-group/another-project + def external_project_path? + ci_config_path =~ /\A.+(yml|yaml)@.+\z/ + end + + # Example: path/to/.gitlab-ci.yml@another-group/another-project:refname + def extract_location_tokens + path_file, path_project = ci_config_path.split('@', 2) + + if path_project.include? ":" + project, ref = path_project.split(':', 2) + [path_file, project, ref] + else + [path_file, path_project] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/parameter.rb b/lib/gitlab/ci/project_config/parameter.rb new file mode 100644 index 00000000000..69e699c27f1 --- /dev/null +++ b/lib/gitlab/ci/project_config/parameter.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Parameter < Source + def content + strong_memoize(:content) do + next unless custom_content.present? + + custom_content + end + end + + def source + :parameter_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/remote.rb b/lib/gitlab/ci/project_config/remote.rb new file mode 100644 index 00000000000..cf1292706d2 --- /dev/null +++ b/lib/gitlab/ci/project_config/remote.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Remote < Source + def content + strong_memoize(:content) do + next unless ci_config_path =~ URI::DEFAULT_PARSER.make_regexp(%w[http https]) + + YAML.dump('include' => [{ 'remote' => ci_config_path }]) + end + end + + def source + :remote_source + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb new file mode 100644 index 00000000000..435ad4d42fe --- /dev/null +++ b/lib/gitlab/ci/project_config/repository.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Repository < Source + def content + strong_memoize(:content) do + next unless file_in_repository? + + YAML.dump('include' => [{ 'local' => ci_config_path }]) + end + end + + def source + :repository_source + end + + private + + def file_in_repository? + return unless project + return unless sha + + project.repository.gitlab_ci_yml_for(sha, ci_config_path).present? + rescue GRPC::NotFound, GRPC::Internal + nil + end + end + end + end +end diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb new file mode 100644 index 00000000000..ebe5728163b --- /dev/null +++ b/lib/gitlab/ci/project_config/source.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class ProjectConfig + class Source + include Gitlab::Utils::StrongMemoize + + def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) + @project = project + @sha = sha + @custom_content = custom_content + @pipeline_source = pipeline_source + @pipeline_source_bridge = pipeline_source_bridge + end + + def exists? + strong_memoize(:exists) do + content.present? + end + end + + def content + raise NotImplementedError + end + + def source + raise NotImplementedError + end + + private + + attr_reader :project, :sha, :custom_content, :pipeline_source, :pipeline_source_bridge + + def ci_config_path + @ci_config_path ||= project.ci_config_path_or_default + end + end + end + end +end diff --git a/lib/gitlab/ci/reports/coverage_report_generator.rb b/lib/gitlab/ci/reports/coverage_report_generator.rb index 6d57e05aa63..88b3b14d5c9 100644 --- a/lib/gitlab/ci/reports/coverage_report_generator.rb +++ b/lib/gitlab/ci/reports/coverage_report_generator.rb @@ -35,7 +35,7 @@ module Gitlab private def report_builds - @pipeline.latest_report_builds_in_self_and_descendants(::Ci::JobArtifact.coverage_reports) + @pipeline.latest_report_builds_in_self_and_project_descendants(::Ci::JobArtifact.of_report_type(:coverage)) end end end diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 86b9be274cc..198b34451b4 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -7,10 +7,10 @@ module Gitlab class Component attr_reader :component_type, :name, :version - def initialize(component = {}) - @component_type = component['type'] - @name = component['name'] - @version = component['version'] + def initialize(type:, name:, version:) + @component_type = type + @name = name + @version = version end end end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index dc6b3153e51..4f84d12f78c 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -17,11 +17,11 @@ module Gitlab end def set_source(source) - self.source = Source.new(source) + self.source = source end def add_component(component) - components << Component.new(component) + components << component end private diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index 60bf30b65a5..ea0fb8d4fbb 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -7,10 +7,10 @@ module Gitlab class Source attr_reader :source_type, :data, :fingerprint - def initialize(source = {}) - @source_type = source['type'] - @data = source['data'] - @fingerprint = source['fingerprint'] + def initialize(type:, data:, fingerprint:) + @source_type = type + @data = data + @fingerprint = fingerprint end end end diff --git a/lib/gitlab/ci/reports/security/scanner.rb b/lib/gitlab/ci/reports/security/scanner.rb index 1ac66a0c671..918df163ede 100644 --- a/lib/gitlab/ci/reports/security/scanner.rb +++ b/lib/gitlab/ci/reports/security/scanner.rb @@ -7,13 +7,13 @@ module Gitlab class Scanner ANALYZER_ORDER = { "bundler_audit" => 1, - "retire.js" => 2, + "retire.js" => 2, "gemnasium" => 3, "gemnasium-maven" => 3, "gemnasium-python" => 3, "bandit" => 1, "spotbugs" => 1, - "semgrep" => 2 + "semgrep" => 2 }.freeze attr_accessor :external_id, :name, :vendor, :version diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb index 5d60aa8f540..a136044c124 100644 --- a/lib/gitlab/ci/status/build/failed.rb +++ b/lib/gitlab/ci/status/build/failed.rb @@ -31,6 +31,7 @@ module Gitlab downstream_pipeline_creation_failed: 'downstream pipeline can not be created', secrets_provider_not_found: 'secrets provider can not be found', reached_max_descendant_pipelines_depth: 'reached maximum depth of child pipelines', + reached_max_pipeline_hierarchy_size: 'downstream pipeline tree is too large', project_deleted: 'pipeline project was deleted', user_blocked: 'pipeline user was blocked', ci_quota_exceeded: 'no more CI minutes available', @@ -39,7 +40,8 @@ module Gitlab builds_disabled: 'project builds are disabled', environment_creation_failure: 'environment creation failure', deployment_rejected: 'deployment rejected', - ip_restriction_failure: 'IP address restriction failure' + ip_restriction_failure: 'IP address restriction failure', + failed_outdated_deployment_job: 'failed outdated deployment job' }.freeze private_constant :REASONS diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index f0ddc4b4916..539e1a6385d 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..70f85382967 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,244 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/Dependency-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/ +# +# Configure dependency scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/dependency_scanning/index.html#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + DS_EXCLUDED_ANALYZERS: "" + DS_EXCLUDED_PATHS: "spec, test, tests, tmp" + DS_MAJOR_VERSION: 3 + +dependency_scanning: + stage: test + script: + - echo "$CI_JOB_NAME is used for configuration only, and its script should not be executed" + - exit 1 + artifacts: + reports: + dependency_scanning: gl-dependency-scanning-report.json + dependencies: [] + rules: + - when: never + +.ds-analyzer: + extends: dependency_scanning + allow_failure: true + variables: + # DS_ANALYZER_IMAGE is an undocumented variable used internally to allow QA to + # override the analyzer image with a custom value. This may be subject to change or + # breakage across GitLab releases. + DS_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/$DS_ANALYZER_NAME:$DS_MAJOR_VERSION" + # DS_ANALYZER_NAME is an undocumented variable used in job definitions + # to inject the analyzer name in the image name. + DS_ANALYZER_NAME: "" + image: + name: "$DS_ANALYZER_IMAGE$DS_IMAGE_SUFFIX" + # `rules` must be overridden explicitly by each child job + # see https://gitlab.com/gitlab-org/gitlab/-/issues/218444 + script: + - /analyzer run + +.cyclonedx-reports: + artifacts: + paths: + - "**/gl-sbom-*.cdx.json" + +.gemnasium-shared-rule: + exists: + - '{Gemfile.lock,*/Gemfile.lock,*/*/Gemfile.lock}' + - '{composer.lock,*/composer.lock,*/*/composer.lock}' + - '{gems.locked,*/gems.locked,*/*/gems.locked}' + - '{go.sum,*/go.sum,*/*/go.sum}' + - '{npm-shrinkwrap.json,*/npm-shrinkwrap.json,*/*/npm-shrinkwrap.json}' + - '{package-lock.json,*/package-lock.json,*/*/package-lock.json}' + - '{yarn.lock,*/yarn.lock,*/*/yarn.lock}' + - '{packages.lock.json,*/packages.lock.json,*/*/packages.lock.json}' + - '{conan.lock,*/conan.lock,*/*/conan.lock}' + +gemnasium-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium" + GEMNASIUM_LIBRARY_SCAN_ENABLED: "true" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium([^-]|$)/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-shared-rule, exists] + +.gemnasium-maven-shared-rule: + exists: + - '{build.gradle,*/build.gradle,*/*/build.gradle}' + - '{build.gradle.kts,*/build.gradle.kts,*/*/build.gradle.kts}' + - '{build.sbt,*/build.sbt,*/*/build.sbt}' + - '{pom.xml,*/pom.xml,*/*/pom.xml}' + +gemnasium-maven-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-maven" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-maven/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + DS_REMEDIATE: "false" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-maven-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-maven-shared-rule, exists] + +.gemnasium-python-shared-rule: + exists: + - '{requirements.txt,*/requirements.txt,*/*/requirements.txt}' + - '{requirements.pip,*/requirements.pip,*/*/requirements.pip}' + - '{Pipfile,*/Pipfile,*/*/Pipfile}' + - '{requires.txt,*/requires.txt,*/*/requires.txt}' + - '{setup.py,*/setup.py,*/*/setup.py}' + - '{poetry.lock,*/poetry.lock,*/*/poetry.lock}' + +gemnasium-python-dependency_scanning: + extends: + - .ds-analyzer + - .cyclonedx-reports + variables: + DS_ANALYZER_NAME: "gemnasium-python" + rules: + - if: $DEPENDENCY_SCANNING_DISABLED + when: never + - if: $DS_EXCLUDED_ANALYZERS =~ /gemnasium-python/ + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $CI_GITLAB_FIPS_MODE == "true" + exists: !reference [.gemnasium-python-shared-rule, exists] + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ + exists: !reference [.gemnasium-python-shared-rule, exists] + # Support passing of $PIP_REQUIREMENTS_FILE + # See https://docs.gitlab.com/ee/user/application_security/dependency_scanning/#configuring-specific-analyzers-used-by-dependency-scanning + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE && + $CI_GITLAB_FIPS_MODE == "true" + variables: + DS_IMAGE_SUFFIX: "-fips" + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\bdependency_scanning\b/ && + $PIP_REQUIREMENTS_FILE + +bundler-audit-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "bundler-audit" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/347491" + - exit 1 + rules: + - when: never + +retire-js-dependency_scanning: + extends: .ds-analyzer + variables: + DS_ANALYZER_NAME: "retire.js" + DS_MAJOR_VERSION: 2 + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.0" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/289830" + - exit 1 + rules: + - when: never diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 1a2a8b4edb4..78fe108e8b9 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index cb8818357a2..bc2e1fed0d4 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.33.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.37.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..e47f669c2e2 --- /dev/null +++ b/lib/gitlab/ci/templates/Jobs/License-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,48 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Jobs/License-Scanning.gitlab-ci.yml + +# Read more about this feature here: https://docs.gitlab.com/ee/user/compliance/license_compliance/index.html +# +# Configure license scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/compliance/license_compliance/#available-variables + +variables: + # Setting this variable will affect all Security templates + # (SAST, Dependency Scanning, ...) + SECURE_ANALYZERS_PREFIX: "$CI_TEMPLATE_REGISTRY_HOST/security-products" + + LICENSE_MANAGEMENT_SETUP_CMD: '' # If needed, specify a command to setup your environment with a custom package manager. + LICENSE_MANAGEMENT_VERSION: 4 + +license_scanning: + stage: test + image: + name: "$SECURE_ANALYZERS_PREFIX/license-finder:$LICENSE_MANAGEMENT_VERSION" + entrypoint: [""] + variables: + LM_REPORT_VERSION: '2.1' + SETUP_CMD: $LICENSE_MANAGEMENT_SETUP_CMD + allow_failure: true + script: + - /run.sh analyze . + artifacts: + reports: + license_scanning: gl-license-scanning-report.json + dependencies: [] + rules: + - if: $LICENSE_MANAGEMENT_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $GITLAB_FEATURES =~ /\blicense_scanning\b/ diff --git a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml index dd164c00724..a6d47e31de2 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.gitlab-ci.yml @@ -36,19 +36,12 @@ sast: bandit-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.py' + - when: never brakeman-sast: extends: .sast-analyzer @@ -69,23 +62,12 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' + - when: never flawfinder-sast: extends: .sast-analyzer @@ -125,19 +107,12 @@ kubesec-sast: gosec-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 3 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.4" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_COMMIT_BRANCH - exists: - - '**/*.go' + - when: never .mobsf-sast: extends: .sast-analyzer @@ -261,6 +236,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.cs' + - '**/*.html' sobelow-sast: extends: .sast-analyzer @@ -297,6 +274,5 @@ spotbugs-sast: - if: $CI_COMMIT_BRANCH exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index c6938920ea4..c0ca821ebff 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -36,24 +36,12 @@ sast: bandit-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/bandit:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.3" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /bandit/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.py' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.py' + - when: never brakeman-sast: extends: .sast-analyzer @@ -80,32 +68,12 @@ brakeman-sast: eslint-sast: extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 2 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/eslint:$SAST_ANALYZER_IMAGE_TAG" + script: + - echo "This job was deprecated in GitLab 14.8 and removed in GitLab 15.3" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /eslint/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.html' - - '**/*.js' - - '**/*.jsx' - - '**/*.ts' - - '**/*.tsx' + - when: never flawfinder-sast: extends: .sast-analyzer @@ -138,6 +106,15 @@ flawfinder-sast: - '**/*.cp' - '**/*.cxx' +gosec-sast: + extends: .sast-analyzer + script: + - echo "This job was deprecated in GitLab 15.0 and removed in GitLab 15.2" + - echo "For more information see https://gitlab.com/gitlab-org/gitlab/-/issues/352554" + - exit 1 + rules: + - when: never + kubesec-sast: extends: .sast-analyzer image: @@ -159,27 +136,6 @@ kubesec-sast: - if: $CI_COMMIT_BRANCH && $SCAN_KUBERNETES_MANIFESTS == 'true' -gosec-sast: - extends: .sast-analyzer - image: - name: "$SAST_ANALYZER_IMAGE" - variables: - SAST_ANALYZER_IMAGE_TAG: 3 - SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/gosec:$SAST_ANALYZER_IMAGE_TAG" - rules: - - if: $SAST_DISABLED - when: never - - if: $SAST_EXCLUDED_ANALYZERS =~ /gosec/ - when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - exists: - - '**/*.go' - - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. - when: never - - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. - exists: - - '**/*.go' - .mobsf-sast: extends: .sast-analyzer image: @@ -323,7 +279,7 @@ semgrep-sast: image: name: "$SAST_ANALYZER_IMAGE" variables: - SERACH_MAX_DEPTH: 20 + SEARCH_MAX_DEPTH: 20 SAST_ANALYZER_IMAGE_TAG: 3 SAST_ANALYZER_IMAGE: "$SECURE_ANALYZERS_PREFIX/semgrep:$SAST_ANALYZER_IMAGE_TAG$SAST_IMAGE_SUFFIX" rules: @@ -341,6 +297,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.html' + - '**/*.cs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. @@ -353,6 +311,8 @@ semgrep-sast: - '**/*.c' - '**/*.go' - '**/*.java' + - '**/*.html' + - '**/*.cs' sobelow-sast: extends: .sast-analyzer @@ -394,7 +354,6 @@ spotbugs-sast: - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -402,6 +361,5 @@ spotbugs-sast: - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. exists: - '**/*.groovy' - - '**/*.java' - '**/*.scala' - '**/*.kt' diff --git a/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml b/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml new file mode 100644 index 00000000000..c8939c8f5a2 --- /dev/null +++ b/lib/gitlab/ci/templates/Katalon.gitlab-ci.yml @@ -0,0 +1,65 @@ +# This template is provided and maintained by Katalon, an official Technology Partner with GitLab. +# +# Use this template to run a Katalon Studio test from this repository. +# You can: +# - Copy and paste this template into a new `.gitlab-ci.yml` file. +# - Add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword. +# +# In either case, you must also select which job you want to run, `.katalon_tests` +# or `.katalon_tests_with_artifacts` (see configuration below), and add that configuration +# to a new job with `extends:`. For example: +# +# Katalon-tests: +# extends: +# - .katalon_tests_with_artifacts +# +# Requirements: +# - A Katalon Studio project with the content saved in the root GitLab repository folder. +# - An active KRE license. +# - A valid Katalon API key. +# +# CI/CD variables, set in the project CI/CD settings: +# - KATALON_TEST_SUITE_PATH: The default path is `Test Suites/<Your Test Suite Name>`. +# Defines which test suite to run. +# - KATALON_API_KEY: The Katalon API key. +# - KATALON_PROJECT_DIR: Optional. Add if the project is in another location. +# - KATALON_ORG_ID: Optional. Add if you are part of multiple Katalon orgs. +# Set to the Org ID that has KRE licenses assigned. For more info on the Org ID, +# see https://support.katalon.com/hc/en-us/articles/4724459179545-How-to-get-Organization-ID- + +.katalon_tests: + # Use the latest version of the Katalon Runtime Engine. You can also use other versions of the + # Katalon Runtime Engine by specifying another tag, for example `katalonstudio/katalon:8.1.2` + # or `katalonstudio/katalon:8.3.0`. + image: 'katalonstudio/katalon' + services: + - docker:dind + variables: + # Specify the Katalon Studio project directory. By default, it is stored under the root project folder. + KATALON_PROJECT_DIR: $CI_PROJECT_DIR + + # The following bash script has two different versions, one if you set the KATALON_ORG_ID + # CI/CD variable, and the other if you did not set it. If you have more than one org in + # admin.katalon.com you must set the KATALON_ORG_ID variable with an ORG ID or + # the Katalon Test Suite fails to run. + # + # You can update or add additional `katalonc` commands below. To see all of the arguments + # `katalonc` supports, go to https://docs.katalon.com/katalon-studio/docs/console-mode-execution.html + script: + - |- + if [[ $KATALON_ORG_ID == "" ]]; then + katalonc.sh -projectPath=$KATALON_PROJECT_DIR -apiKey=$KATALON_API_KEY -browserType="Chrome" -retry=0 -statusDelay=20 -testSuitePath="$KATALON_TEST_SUITE_PATH" -reportFolder=Reports/ + else + katalonc.sh -projectPath=$KATALON_PROJECT_DIR -apiKey=$KATALON_API_KEY -browserType="Chrome" -retry=0 -statusDelay=20 -orgID=$KATALON_ORG_ID -testSuitePath="$KATALON_TEST_SUITE_PATH" -reportFolder=Reports/ + fi + +# Upload the artifacts and make the junit report accessible under the Pipeline Tests +.katalon_tests_with_artifacts: + extends: .katalon_tests + artifacts: + when: always + paths: + - Reports/ + reports: + junit: + Reports/*/*/*/*.xml diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml index 3d7883fb87a..79a08c33fdf 100644 --- a/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml @@ -11,12 +11,12 @@ # # Requirements: # - A `test` stage to be present in the pipeline. -# - You must define the image to be scanned in the DOCKER_IMAGE variable. If DOCKER_IMAGE is the +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the # same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. -# - Container registry credentials defined by `DOCKER_USER` and `DOCKER_PASSWORD` variables if the +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the # image to be scanned is in a private registry. # - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the -# DOCKERFILE_PATH variable. +# CS_DOCKERFILE_PATH variable. # # Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). # List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables diff --git a/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml new file mode 100644 index 00000000000..f7b1d12b3b3 --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Container-Scanning.latest.gitlab-ci.yml @@ -0,0 +1,68 @@ +# To contribute improvements to CI/CD templates, please follow the Development guide at: +# https://docs.gitlab.com/ee/development/cicd/templates.html +# This specific template is located at: +# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/Container-Scanning.gitlab-ci.yml + +# Use this template to enable container scanning in your project. +# You should add this template to an existing `.gitlab-ci.yml` file by using the `include:` +# keyword. +# The template should work without modifications but you can customize the template settings if +# needed: https://docs.gitlab.com/ee/user/application_security/container_scanning/#customizing-the-container-scanning-settings +# +# Requirements: +# - A `test` stage to be present in the pipeline. +# - You must define the image to be scanned in the CS_IMAGE variable. If CS_IMAGE is the +# same as $CI_APPLICATION_REPOSITORY:$CI_APPLICATION_TAG, you can skip this. +# - Container registry credentials defined by `CS_REGISTRY_USER` and `CS_REGISTRY_PASSWORD` variables if the +# image to be scanned is in a private registry. +# - For auto-remediation, a readable Dockerfile in the root of the project or as defined by the +# CS_DOCKERFILE_PATH variable. +# +# Configure container scanning with CI/CD variables (https://docs.gitlab.com/ee/ci/variables/index.html). +# List of available variables: https://docs.gitlab.com/ee/user/application_security/container_scanning/#available-variables + +variables: + CS_ANALYZER_IMAGE: "$CI_TEMPLATE_REGISTRY_HOST/security-products/container-scanning:5" + +container_scanning: + image: "$CS_ANALYZER_IMAGE$CS_IMAGE_SUFFIX" + stage: test + variables: + # To provide a `vulnerability-allowlist.yml` file, override the GIT_STRATEGY variable in your + # `.gitlab-ci.yml` file and set it to `fetch`. + # For details, see the following links: + # https://docs.gitlab.com/ee/user/application_security/container_scanning/index.html#overriding-the-container-scanning-template + # https://docs.gitlab.com/ee/user/application_security/container_scanning/#vulnerability-allowlisting + GIT_STRATEGY: none + allow_failure: true + artifacts: + reports: + container_scanning: gl-container-scanning-report.json + dependency_scanning: gl-dependency-scanning-report.json + paths: [gl-container-scanning-report.json, gl-dependency-scanning-report.json] + dependencies: [] + script: + - gtcs scan + rules: + - if: $CONTAINER_SCANNING_DISABLED + when: never + + # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + + # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + - if: $CI_OPEN_MERGE_REQUESTS + when: never + + # Add the job to branch pipelines. + - if: $CI_COMMIT_BRANCH && + $CI_GITLAB_FIPS_MODE == "true" && + $CS_ANALYZER_IMAGE !~ /-(fips|ubi)\z/ + variables: + CS_IMAGE_SUFFIX: -fips + - if: $CI_COMMIT_BRANCH diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 3a956ebfc49..9a40a23b276 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -9,7 +9,7 @@ # There is a more opinionated template which we suggest the users to abide, # which is the lib/gitlab/ci/templates/Terraform.gitlab-ci.yml image: - name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/terraform:1.1.9" + name: "$CI_TEMPLATE_REGISTRY_HOST/gitlab-org/terraform-images/releases/1.1:v0.43.0" variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index 64c784f43cb..fb0d300338b 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -38,7 +38,7 @@ publish: # Compare the version in package.json to all published versions. # If the package.json version has not yet been published, run `npm publish`. - | - if [[ $(npm view "${NPM_PACKAGE_NAME}" versions) != *"'${NPM_PACKAGE_VERSION}'"* ]]; then + if [[ "$(npm view ${NPM_PACKAGE_NAME} versions)" != *"'${NPM_PACKAGE_VERSION}'"* ]]; then npm publish echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" else diff --git a/lib/gitlab/ci/trace.rb b/lib/gitlab/ci/trace.rb index 95a60b852b8..c5664ef1cfb 100644 --- a/lib/gitlab/ci/trace.rb +++ b/lib/gitlab/ci/trace.rb @@ -23,7 +23,6 @@ module Gitlab attr_reader :job - delegate :old_trace, to: :job delegate :can_attempt_archival_now?, :increment_archival_attempts!, :archival_attempts_message, :archival_attempts_available?, to: :trace_metadata @@ -82,7 +81,7 @@ module Gitlab end def live? - job.trace_chunks.any? || current_path.present? || old_trace.present? + job.trace_chunks.any? || current_path.present? end def read(&block) @@ -111,7 +110,6 @@ module Gitlab # Erase the live trace erase_trace_chunks! FileUtils.rm_f(current_path) if current_path # Remove a trace file of a live trace - job.erase_old_trace! if job.has_old_trace? # Remove a trace in database of a live trace ensure @current_path = nil end @@ -162,8 +160,6 @@ module Gitlab Gitlab::Ci::Trace::ChunkedIO.new(job) elsif current_path File.open(current_path, "rb") - elsif old_trace - StringIO.new(old_trace) end end @@ -210,11 +206,6 @@ module Gitlab archive_stream!(stream) FileUtils.rm(current_path) end - elsif old_trace - StringIO.new(old_trace, 'rb').tap do |stream| - archive_stream!(stream) - job.erase_old_trace! - end end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index 95dff83506d..528d72c9bcc 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -118,7 +118,7 @@ module Gitlab def predefined_variables(job) Gitlab::Ci::Variables::Collection.new.tap do |variables| variables.append(key: 'CI_JOB_NAME', value: job.name) - variables.append(key: 'CI_JOB_STAGE', value: job.stage) + variables.append(key: 'CI_JOB_STAGE', value: job.stage_name) variables.append(key: 'CI_JOB_MANUAL', value: 'true') if job.action? variables.append(key: 'CI_PIPELINE_TRIGGERED', value: 'true') if job.trigger_request @@ -127,7 +127,7 @@ module Gitlab # legacy variables variables.append(key: 'CI_BUILD_NAME', value: job.name) - variables.append(key: 'CI_BUILD_STAGE', value: job.stage) + variables.append(key: 'CI_BUILD_STAGE', value: job.stage_name) variables.append(key: 'CI_BUILD_TRIGGERED', value: 'true') if job.trigger_request variables.append(key: 'CI_BUILD_MANUAL', value: 'true') if job.action? end diff --git a/lib/gitlab/ci/variables/helpers.rb b/lib/gitlab/ci/variables/helpers.rb index 7cc727bb3ea..16e3afd8620 100644 --- a/lib/gitlab/ci/variables/helpers.rb +++ b/lib/gitlab/ci/variables/helpers.rb @@ -6,24 +6,24 @@ module Gitlab module Helpers class << self def merge_variables(current_vars, new_vars) - current_vars = transform_from_yaml_variables(current_vars) - new_vars = transform_from_yaml_variables(new_vars) + return current_vars if new_vars.blank? - transform_to_yaml_variables( - current_vars.merge(new_vars) - ) - end + current_vars = transform_to_array(current_vars) if current_vars.is_a?(Hash) + new_vars = transform_to_array(new_vars) if new_vars.is_a?(Hash) - def transform_to_yaml_variables(vars) - vars.to_h.map do |key, value| - { key: key.to_s, value: value, public: true } - end + (new_vars + current_vars).uniq { |var| var[:key] } end - def transform_from_yaml_variables(vars) - return vars.stringify_keys.transform_values(&:to_s) if vars.is_a?(Hash) + def transform_to_array(vars) + return [] if vars.blank? - vars.to_a.to_h { |var| [var[:key].to_s, var[:value]] } + vars.map do |key, data| + if data.is_a?(Hash) + { key: key.to_s, **data.except(:key) } + else + { key: key.to_s, value: data } + end + end end def inherit_yaml_variables(from:, to:, inheritance:) @@ -35,7 +35,7 @@ module Gitlab def apply_inheritance(variables, inheritance) case inheritance when true then variables - when false then {} + when false then [] when Array then variables.select { |var| inheritance.include?(var[:key]) } end end diff --git a/lib/gitlab/ci/yaml_processor/feature_flags.rb b/lib/gitlab/ci/yaml_processor/feature_flags.rb index f03db9d0e6b..50d37f6e4a0 100644 --- a/lib/gitlab/ci/yaml_processor/feature_flags.rb +++ b/lib/gitlab/ci/yaml_processor/feature_flags.rb @@ -5,10 +5,10 @@ module Gitlab class YamlProcessor module FeatureFlags ACTOR_KEY = 'ci_yaml_processor_feature_flag_actor' + CORRECT_USAGE_KEY = 'ci_yaml_processor_feature_flag_correct_usage' NO_ACTOR_VALUE = :no_actor - - NoActorError = Class.new(StandardError) NO_ACTOR_MESSAGE = "Actor not set. Ensure to call `enabled?` inside `with_actor` block" + NoActorError = Class.new(StandardError) class << self # Cache a feature flag actor as thread local variable so @@ -31,6 +31,15 @@ module Gitlab ::Feature.enabled?(feature_flag, current_actor) end + def ensure_correct_usage + previous = Thread.current[CORRECT_USAGE_KEY] + Thread.current[CORRECT_USAGE_KEY] = true + + yield + ensure + Thread.current[CORRECT_USAGE_KEY] = previous + end + private def current_actor @@ -39,10 +48,22 @@ module Gitlab value rescue NoActorError => e - Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + handle_missing_actor(e) nil end + + def handle_missing_actor(exception) + if ensure_correct_usage? + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(exception) + else + Gitlab::ErrorTracking.track_exception(exception) + end + end + + def ensure_correct_usage? + Thread.current[CORRECT_USAGE_KEY] == true + end end end end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 4bd1ac3b67f..f203f88442d 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -43,7 +43,7 @@ module Gitlab end def root_variables - @root_variables ||= transform_to_yaml_variables(variables) + @root_variables ||= transform_to_array(variables) end def jobs @@ -70,7 +70,7 @@ module Gitlab environment: job[:environment_name], coverage_regex: job[:coverage], # yaml_variables is calculated with using job_variables in Seed::Build - job_variables: transform_to_yaml_variables(job[:job_variables]), + job_variables: transform_to_array(job[:job_variables]), root_variables_inheritance: job[:root_variables_inheritance], needs_attributes: job.dig(:needs, :job), interruptible: job[:interruptible], @@ -114,7 +114,7 @@ module Gitlab Gitlab::Ci::Variables::Helpers.inherit_yaml_variables( from: root_variables, - to: transform_to_yaml_variables(job[:job_variables]), + to: job[:job_variables], inheritance: job.fetch(:root_variables_inheritance, true) ) end @@ -137,8 +137,8 @@ module Gitlab job[:release] end - def transform_to_yaml_variables(variables) - ::Gitlab::Ci::Variables::Helpers.transform_to_yaml_variables(variables) + def transform_to_array(variables) + ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end end end diff --git a/lib/gitlab/cleanup/personal_access_tokens.rb b/lib/gitlab/cleanup/personal_access_tokens.rb new file mode 100644 index 00000000000..a1e4b5765c2 --- /dev/null +++ b/lib/gitlab/cleanup/personal_access_tokens.rb @@ -0,0 +1,105 @@ +# frozen_string_literal: true + +module Gitlab + module Cleanup + class PersonalAccessTokens + # By default tokens that haven't been used for over 1 year will be revoked + DEFAULT_TIME_PERIOD = 1.year + # To prevent inadvertently revoking all tokens, we provide a minimum time + MINIMUM_TIME_PERIOD = 1.day + + attr_reader :logger, :cut_off_date, :revocation_time, :group + + def initialize(cut_off_date: DEFAULT_TIME_PERIOD.ago.beginning_of_day, logger: nil, group_full_path:) + @cut_off_date = cut_off_date + + # rubocop: disable CodeReuse/ActiveRecord + @group = Group.find_by_full_path(group_full_path) + # rubocop: enable CodeReuse/ActiveRecord + + raise "Group with full_path #{group_full_path} not found" unless @group + raise "Invalid time: #{@cut_off_date}" unless @cut_off_date <= MINIMUM_TIME_PERIOD.ago + + # Use a static revocation time to make correlation of revoked + # tokens easier, should it be needed. + @revocation_time = Time.current.utc + @logger = logger || Gitlab::AppJsonLogger + + raise "Invalid logger: #{@logger}" unless @logger.respond_to?(:info) && @logger.respond_to?(:warn) + end + + def run!(dry_run: true, revoke_active_tokens: false) + # rubocop:disable Rails/Output + if dry_run + puts "Dry running. No changes will be made" + elsif revoke_active_tokens + puts "Revoking used and unused access tokens created before #{cut_off_date}..." + else + puts "Revoking access tokens last used and created before #{cut_off_date}..." + end + # rubocop:enable Rails/Output + + tokens_to_revoke = revocable_tokens(revoke_active_tokens) + + # rubocop:disable Cop/InBatches + tokens_to_revoke.in_batches do |access_tokens| + revoke_batch(access_tokens, dry_run) + end + # rubocop:enable Cop/InBatches + end + + private + + def revocable_tokens(revoke_active_tokens) + if revoke_active_tokens + PersonalAccessToken + .active + .owner_is_human + .created_before(cut_off_date) + .for_users(group.users) + else + PersonalAccessToken + .active + .owner_is_human + .last_used_before_or_unused(cut_off_date) + .for_users(group.users) + end + end + + def revoke_batch(access_tokens, dry_run) + # Capture a simplified set of attributes for logging and for + # determining when an error has led some records to not be + # updated + attrs = access_tokens.as_json(only: [:id, :user_id]) + + # Use `update_all` to bypass any validations which might + # prevent revocation. Manually specify updated_at. + affected_row_count = dry_run ? 0 : access_tokens.update_all(revoked: true, updated_at: @revocation_time) + + message = { + dry_run: dry_run, + message: "Revoke token batch", + token_count: attrs.size, + updated_count: affected_row_count, + tokens: attrs, + group_full_path: group.full_path + } + + # rubocop:disable Rails/Output + if dry_run + puts "Dry run complete. #{attrs.size} rows would be affected" + logger.info(message) + elsif affected_row_count.eql?(attrs.size) + puts "Finished. #{attrs.size} rows affected" + logger.info(message) + else + # :nocov: + puts "ERROR. #{affected_row_count} tokens deleted, #{attrs.size} tokens should have been deleted" + logger.warn(message) + # :nocov: + end + # rubocop:enable Rails/Output + end + end + end +end diff --git a/lib/gitlab/closing_issue_extractor.rb b/lib/gitlab/closing_issue_extractor.rb index 8e624215065..7104de2a3c3 100644 --- a/lib/gitlab/closing_issue_extractor.rb +++ b/lib/gitlab/closing_issue_extractor.rb @@ -17,7 +17,6 @@ module Gitlab def closed_by_message(message) return [] if message.nil? - return [] unless @project.autoclose_referenced_issues closing_statements = [] message.scan(ISSUE_CLOSING_REGEX) do @@ -27,8 +26,9 @@ module Gitlab @extractor.analyze(closing_statements.join(" ")) @extractor.issues.reject do |issue| - # Don't extract issues from the project this project was forked from - @extractor.project.forked_from?(issue.project) + @extractor.project.forked_from?(issue.project) || + !issue.project.autoclose_referenced_issues || + !issue.project.issues_enabled? end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index e423d1f17da..be08ada9d2f 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -4,6 +4,11 @@ require_relative '../utils' # Gitlab::Utils module Gitlab module Cluster + # We take advantage of the fact that the application is pre-loaded in the primary + # process. If it's a pre-fork server like Puma, this will be the Puma master process. + # Otherwise it is the worker itself such as for Sidekiq. + PRIMARY_PID = $$ + # # LifecycleEvents lets Rails initializers register application startup hooks # that are sensitive to forking. For example, to defer the creation of diff --git a/lib/gitlab/config/entry/composable_hash.rb b/lib/gitlab/config/entry/composable_hash.rb index 9531b7e56fd..0b892fd4552 100644 --- a/lib/gitlab/config/entry/composable_hash.rb +++ b/lib/gitlab/config/entry/composable_hash.rb @@ -25,9 +25,9 @@ module Gitlab entry_class_name = entry_class.name.demodulize.underscore factory = ::Gitlab::Config::Entry::Factory.new(entry_class) - .value(config || {}) + .value(config.nil? ? {} : config) .with(key: name, parent: self, description: "#{name} #{entry_class_name} definition") # rubocop:disable CodeReuse/ActiveRecord - .metadata(name: name) + .metadata(composable_metadata.merge(name: name)) @entries[name] = factory.create! end @@ -38,9 +38,15 @@ module Gitlab end end + private + def composable_class(name, config) opt(:composable_class) end + + def composable_metadata + {} + end end end end diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index cc24ae837f3..337cfbc5287 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -304,6 +304,7 @@ module Gitlab end end + # This will be removed with the FF `ci_variables_refactoring_to_variable`. class VariablesValidator < ActiveModel::EachValidator include LegacyValidationHelpers @@ -336,6 +337,18 @@ module Gitlab end end + class AlphanumericValidator < ActiveModel::EachValidator + def self.validate(value) + value.is_a?(String) || value.is_a?(Symbol) || value.is_a?(Integer) + end + + def validate_each(record, attribute, value) + unless self.class.validate(value) + record.errors.add(attribute, 'must be an alphanumeric string') + end + end + end + class ExpressionValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) unless value.is_a?(String) && ::Gitlab::Ci::Pipeline::Expression::Statement.new(value).valid? diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb index ff457fb9219..47a6e67a5a1 100644 --- a/lib/gitlab/container_repository/tags/cache.rb +++ b/lib/gitlab/container_repository/tags/cache.rb @@ -48,14 +48,14 @@ module Gitlab ::Gitlab::Redis::Cache.with do |redis| # we use a pipeline instead of a MSET because each tag has # a specific ttl - redis.pipelined do + redis.pipelined do |pipeline| cacheable_tags.each do |tag| created_at = tag.created_at # ttl is the max_ttl_in_seconds reduced by the number # of seconds that the tag has already existed ttl = max_ttl_in_seconds - (now - created_at).seconds ttl = ttl.to_i - redis.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 + pipeline.set(cache_key(tag), created_at.rfc3339, ex: ttl) if ttl > 0 end end end diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 91e6fc11a53..4640f85bb0a 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -24,7 +24,7 @@ module Gitlab # Leaving this way to have backward compatibility build_id: build.id, build_name: build.name, - build_stage: build.stage, + build_stage: build.stage_name, build_status: build.status, build_created_at: build.created_at, build_started_at: build.started_at, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index 2c124b07006..320ebe5e80f 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -52,7 +52,8 @@ module Gitlab runner: :tags, job_artifacts_archive: [], user: [], - metadata: [] + metadata: [], + ci_stage: [] } } ) @@ -110,7 +111,7 @@ module Gitlab def build_hook_attrs(build) { id: build.id, - stage: build.stage, + stage: build.stage_name, name: build.name, status: build.status, created_at: build.created_at, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 8703365b678..dd84127459d 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -242,7 +242,8 @@ module Gitlab # in such cases it is fine to ignore such connections return unless db_config - primary_model = self.database_base_models.fetch(db_config.name.to_sym) + db_config_name = db_config.name.delete_suffix(LoadBalancing::LoadBalancer::REPLICA_SUFFIX) + primary_model = self.database_base_models.fetch(db_config_name.to_sym) self.schemas_to_base_models.select do |_, child_models| child_models.any? do |child_model| diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 6aed1eed994..45f52765d0f 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -8,6 +8,7 @@ module Gitlab BATCH_CLASS_MODULE = "#{JOB_CLASS_MODULE}::BatchingStrategies" MAXIMUM_FAILED_RATIO = 0.5 MINIMUM_JOBS = 50 + FINISHED_PROGRESS_VALUE = 100 self.table_name = :batched_background_migrations @@ -24,6 +25,7 @@ module Gitlab scope :queue_order, -> { order(id: :asc) } scope :queued, -> { with_statuses(:active, :paused) } + scope :ordered_by_created_at_desc, -> { order(created_at: :desc) } # on_hold_until is a temporary runtime status which puts execution "on hold" scope :executable, -> { with_status(:active).where('on_hold_until IS NULL OR on_hold_until < NOW()') } @@ -57,11 +59,11 @@ module Gitlab state :finalizing, value: 5 event :pause do - transition any => :paused + transition [:active, :paused] => :paused end event :execute do - transition any => :active + transition [:active, :paused, :failed] => :active end event :finish do @@ -231,7 +233,15 @@ module Gitlab "BatchedMigration[id: #{id}]" end + # Computes an estimation of the progress of the migration in percents. + # + # Because `total_tuple_count` is an estimation of the tuples based on DB statistics + # when the migration is complete there can actually be more or less tuples that initially + # estimated as `total_tuple_count` so the progress may not show 100%. For that reason when + # we know migration completed successfully, we just return the 100 value def progress + return FINISHED_PROGRESS_VALUE if finished? + return unless total_tuple_count.to_i > 0 100 * migrated_tuple_count / total_tuple_count diff --git a/lib/gitlab/database/background_migration/health_status.rb b/lib/gitlab/database/background_migration/health_status.rb index 9a283074b32..506d2996ad5 100644 --- a/lib/gitlab/database/background_migration/health_status.rb +++ b/lib/gitlab/database/background_migration/health_status.rb @@ -18,7 +18,7 @@ module Gitlab indicator.new(migration.health_context).evaluate rescue StandardError => e Gitlab::ErrorTracking.track_exception(e, migration_id: migration.id, - job_class_name: migration.job_class_name) + job_class_name: migration.job_class_name) Signals::Unknown.new(indicator, reason: "unexpected error: #{e.message} (#{e.class})") end diff --git a/lib/gitlab/database/batch_average_counter.rb b/lib/gitlab/database/batch_average_counter.rb new file mode 100644 index 00000000000..9cb1e34ab67 --- /dev/null +++ b/lib/gitlab/database/batch_average_counter.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class BatchAverageCounter + COLUMN_FALLBACK = 0 + DEFAULT_BATCH_SIZE = 1_000 + FALLBACK = -1 + MAX_ALLOWED_LOOPS = 10_000 + OFFSET_BY_ONE = 1 + SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep + + attr_reader :relation, :column + + def initialize(relation, column) + @relation = relation + @column = wrap_column(relation, column) + end + + def count(batch_size: nil) + raise 'BatchAverageCounter can not be run inside a transaction' if transaction_open? + + batch_size = batch_size.presence || DEFAULT_BATCH_SIZE + + start = column_start + finish = column_finish + + total_sum = 0 + total_records = 0 + + batch_start = start + + while batch_start < finish + begin + batch_end = [batch_start + batch_size, finish].min + batch_relation = build_relation_batch(batch_start, batch_end) + + # We use `sum` and `count` instead of `average` here to not run into an "average of averages" + # problem as batches will have different sizes, so we are essentially summing up the values for + # each batch separately, and then dividing that result on the total number of records. + batch_sum, batch_count = batch_relation.pick(column.sum, column.count) + + total_sum += batch_sum.to_i + total_records += batch_count + + batch_start = batch_end + rescue ActiveRecord::QueryCanceled => error # rubocop:disable Database/RescueQueryCanceled + # retry with a safe batch size & warmer cache + if batch_size >= 2 * DEFAULT_BATCH_SIZE + batch_size /= 2 + else + log_canceled_batch_fetch(batch_start, batch_relation.to_sql, error) + + return FALLBACK + end + end + + sleep(SLEEP_TIME_IN_SECONDS) + end + + return FALLBACK if total_records == 0 + + total_sum.to_f / total_records + end + + private + + def column_start + relation.unscope(:group, :having).minimum(column) || COLUMN_FALLBACK + end + + def column_finish + (relation.unscope(:group, :having).maximum(column) || COLUMN_FALLBACK) + OFFSET_BY_ONE + end + + def build_relation_batch(start, finish) + relation.where(column.between(start...finish)) + end + + def log_canceled_batch_fetch(batch_start, query, error) + Gitlab::AppJsonLogger + .error( + event: 'batch_count', + relation: relation.table_name, + operation: 'average', + start: batch_start, + query: query, + message: "Query has been canceled with message: #{error.message}" + ) + end + + def transaction_open? + relation.connection.transaction_open? + end + + def wrap_column(relation, column) + return column if column.is_a?(Arel::Attributes::Attribute) + + relation.arel_table[column] + end + end + end +end diff --git a/lib/gitlab/database/batch_count.rb b/lib/gitlab/database/batch_count.rb index 92a41bb36ee..7a064fb4005 100644 --- a/lib/gitlab/database/batch_count.rb +++ b/lib/gitlab/database/batch_count.rb @@ -35,6 +35,10 @@ module Gitlab BatchCounter.new(relation, column: column).count(batch_size: batch_size, start: start, finish: finish) end + def batch_count_with_timeout(relation, column = nil, batch_size: nil, start: nil, finish: nil, timeout: nil, partial_results: nil) + BatchCounter.new(relation, column: column).count_with_timeout(batch_size: batch_size, start: start, finish: finish, timeout: timeout, partial_results: partial_results) + end + def batch_distinct_count(relation, column = nil, batch_size: nil, start: nil, finish: nil) BatchCounter.new(relation, column: column).count(mode: :distinct, batch_size: batch_size, start: start, finish: finish) end @@ -44,7 +48,7 @@ module Gitlab end def batch_average(relation, column, batch_size: nil, start: nil, finish: nil) - BatchCounter.new(relation, column: nil, operation: :average, operation_args: [column]).count(batch_size: batch_size, start: start, finish: finish) + BatchAverageCounter.new(relation, column).count(batch_size: batch_size) end class << self diff --git a/lib/gitlab/database/batch_counter.rb b/lib/gitlab/database/batch_counter.rb index 522b598cd9d..abb62140503 100644 --- a/lib/gitlab/database/batch_counter.rb +++ b/lib/gitlab/database/batch_counter.rb @@ -6,7 +6,6 @@ module Gitlab FALLBACK = -1 MIN_REQUIRED_BATCH_SIZE = 1_250 DEFAULT_SUM_BATCH_SIZE = 1_000 - DEFAULT_AVERAGE_BATCH_SIZE = 1_000 MAX_ALLOWED_LOOPS = 10_000 SLEEP_TIME_IN_SECONDS = 0.01 # 10 msec sleep ALLOWED_MODES = [:itself, :distinct].freeze @@ -27,12 +26,19 @@ module Gitlab def unwanted_configuration?(finish, batch_size, start) (@operation == :count && batch_size <= MIN_REQUIRED_BATCH_SIZE) || (@operation == :sum && batch_size < DEFAULT_SUM_BATCH_SIZE) || - (@operation == :average && batch_size < DEFAULT_AVERAGE_BATCH_SIZE) || (finish - start) / batch_size >= MAX_ALLOWED_LOOPS || start >= finish end def count(batch_size: nil, mode: :itself, start: nil, finish: nil) + result = count_with_timeout(batch_size: batch_size, mode: mode, start: start, finish: finish, timeout: nil) + + return FALLBACK if result[:status] != :completed + + result[:count] + end + + def count_with_timeout(batch_size: nil, mode: :itself, start: nil, finish: nil, timeout: nil, partial_results: nil) raise 'BatchCount can not be run inside a transaction' if transaction_open? check_mode!(mode) @@ -44,12 +50,20 @@ module Gitlab finish = actual_finish(finish) raise "Batch counting expects positive values only for #{@column}" if start < 0 || finish < 0 - return FALLBACK if unwanted_configuration?(finish, batch_size, start) + return { status: :bad_config } if unwanted_configuration?(finish, batch_size, start) - results = nil + results = partial_results batch_start = start + start_time = ::Gitlab::Metrics::System.monotonic_time.seconds + while batch_start < finish + + # Timeout elapsed, return partial result so the caller can continue later + if timeout && ::Gitlab::Metrics::System.monotonic_time.seconds - start_time > timeout + return { status: :timeout, partial_results: results, continue_from: batch_start } + end + begin batch_end = [batch_start + batch_size, finish].min batch_relation = build_relation_batch(batch_start, batch_end, mode) @@ -62,14 +76,14 @@ module Gitlab batch_size /= 2 else log_canceled_batch_fetch(batch_start, mode, batch_relation.to_sql, error) - return FALLBACK + return { status: :cancelled } end end sleep(SLEEP_TIME_IN_SECONDS) end - results + { status: :completed, count: results } end def transaction_open? @@ -94,7 +108,6 @@ module Gitlab def batch_size_for_mode_and_operation(mode, operation) return DEFAULT_SUM_BATCH_SIZE if operation == :sum - return DEFAULT_AVERAGE_BATCH_SIZE if operation == :average mode == :distinct ? DEFAULT_DISTINCT_BATCH_SIZE : DEFAULT_BATCH_SIZE end @@ -132,10 +145,6 @@ module Gitlab message: "Query has been canceled with message: #{error.message}" ) end - - def not_group_by_query? - !@relation.is_a?(ActiveRecord::Relation) || @relation.group_values.blank? - end end end end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index d05eee7d6e6..5725d7a4503 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -91,6 +91,7 @@ ci_job_artifact_states: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci ci_namespace_mirrors: :gitlab_ci +ci_partitions: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -182,6 +183,7 @@ design_management_versions: :gitlab_main design_user_mentions: :gitlab_main detached_partitions: :gitlab_shared diff_note_positions: :gitlab_main +dora_configurations: :gitlab_main dora_daily_metrics: :gitlab_main draft_notes: :gitlab_main elastic_index_settings: :gitlab_main @@ -228,6 +230,7 @@ geo_repository_deleted_events: :gitlab_main geo_repository_renamed_events: :gitlab_main geo_repository_updated_events: :gitlab_main geo_reset_checksum_events: :gitlab_main +ghost_user_migrations: :gitlab_main gitlab_subscription_histories: :gitlab_main gitlab_subscriptions: :gitlab_main gpg_keys: :gitlab_main @@ -315,6 +318,7 @@ merge_request_diff_details: :gitlab_main merge_request_diff_files: :gitlab_main merge_request_diffs: :gitlab_main merge_request_metrics: :gitlab_main +merge_request_predictions: :gitlab_main merge_request_reviewers: :gitlab_main merge_requests_closing_issues: :gitlab_main merge_requests: :gitlab_main @@ -380,6 +384,7 @@ packages_events: :gitlab_main packages_helm_file_metadata: :gitlab_main packages_maven_metadata: :gitlab_main packages_npm_metadata: :gitlab_main +packages_rpm_metadata: :gitlab_main packages_nuget_dependency_link_metadata: :gitlab_main packages_nuget_metadata: :gitlab_main packages_package_file_build_infos: :gitlab_main @@ -399,6 +404,7 @@ plans: :gitlab_main pool_repositories: :gitlab_main postgres_async_indexes: :gitlab_shared postgres_autovacuum_activity: :gitlab_shared +postgres_constraints: :gitlab_shared postgres_foreign_keys: :gitlab_shared postgres_index_bloat_estimates: :gitlab_shared postgres_indexes: :gitlab_shared @@ -479,6 +485,7 @@ sbom_components: :gitlab_main sbom_occurrences: :gitlab_main sbom_component_versions: :gitlab_main sbom_sources: :gitlab_main +sbom_vulnerable_component_versions: :gitlab_main schema_migrations: :gitlab_internal scim_identities: :gitlab_main scim_oauth_access_tokens: :gitlab_main @@ -549,6 +556,7 @@ user_statuses: :gitlab_main user_synced_attributes_metadata: :gitlab_main verification_codes: :gitlab_main vulnerabilities: :gitlab_main +vulnerability_advisories: :gitlab_main vulnerability_exports: :gitlab_main vulnerability_external_issue_links: :gitlab_main vulnerability_feedback: :gitlab_main diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index cd483d616bb..fe75cd763b4 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -5,42 +5,63 @@ module Gitlab class LockWritesManager TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' - def initialize(table_name:, connection:, database_name:, logger: nil) + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @database_name = database_name @logger = logger + @dry_run = dry_run + end + + def table_locked_for_writes?(table_name) + query = <<~SQL + SELECT COUNT(*) from information_schema.triggers + WHERE event_object_table = '#{table_name}' + AND trigger_name = '#{write_trigger_name(table_name)}' + SQL + + connection.select_value(query) == 3 end def lock_writes + if table_locked_for_writes?(table_name) + logger&.info "Skipping lock_writes, because #{table_name} is already locked for writes" + return + end + logger&.info "Database: '#{database_name}', Table: '#{table_name}': Lock Writes".color(:yellow) - sql = <<-SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; + sql_statement = <<~SQL CREATE TRIGGER #{write_trigger_name(table_name)} BEFORE INSERT OR UPDATE OR DELETE OR TRUNCATE ON #{table_name} FOR EACH STATEMENT EXECUTE FUNCTION #{TRIGGER_FUNCTION_NAME}(); SQL - with_retries(connection) do - connection.execute(sql) - end + execute_sql_statement(sql_statement) end def unlock_writes logger&.info "Database: '#{database_name}', Table: '#{table_name}': Allow Writes".color(:green) - sql = <<-SQL - DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name} + sql_statement = <<~SQL + DROP TRIGGER IF EXISTS #{write_trigger_name(table_name)} ON #{table_name}; SQL - with_retries(connection) do - connection.execute(sql) - end + execute_sql_statement(sql_statement) end private - attr_reader :table_name, :connection, :database_name, :logger + attr_reader :table_name, :connection, :database_name, :logger, :dry_run + + def execute_sql_statement(sql) + if dry_run + logger&.info sql + else + with_retries(connection) do + connection.execute(sql) + end + end + end def with_retries(connection, &block) with_statement_timeout_retries do diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index db39524f4f6..e574422ce11 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -936,13 +936,14 @@ module Gitlab def revert_backfill_conversion_of_integer_to_bigint(table, columns, primary_key: :id) columns = Array.wrap(columns) - conditions = ActiveRecord::Base.sanitize_sql([ - 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', - job_class_name: 'CopyColumnUsingBackgroundMigrationJob', - table_name: table, - column_name: primary_key, - job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json - ]) + conditions = ActiveRecord::Base.sanitize_sql( + [ + 'job_class_name = :job_class_name AND table_name = :table_name AND column_name = :column_name AND job_arguments = :job_arguments', + job_class_name: 'CopyColumnUsingBackgroundMigrationJob', + table_name: table, + column_name: primary_key, + job_arguments: [columns, columns.map { |column| convert_to_bigint_column(column) }].to_json + ]) execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end diff --git a/lib/gitlab/database/migrations/base_background_runner.rb b/lib/gitlab/database/migrations/base_background_runner.rb index a9440cafd30..76982a9da9b 100644 --- a/lib/gitlab/database/migrations/base_background_runner.rb +++ b/lib/gitlab/database/migrations/base_background_runner.rb @@ -40,7 +40,7 @@ module Gitlab instrumentation = Instrumentation.new(result_dir: per_background_migration_result_dir) batch_names = (1..).each.lazy.map { |i| "batch_#{i}" } - jobs.shuffle.each do |j| + jobs.each do |j| break if run_until <= Time.current instrumentation.observe(version: nil, diff --git a/lib/gitlab/database/migrations/test_background_runner.rb b/lib/gitlab/database/migrations/test_background_runner.rb index f7713237b38..6da2e098d43 100644 --- a/lib/gitlab/database/migrations/test_background_runner.rb +++ b/lib/gitlab/database/migrations/test_background_runner.rb @@ -15,6 +15,7 @@ module Gitlab def jobs_by_migration_name traditional_background_migrations.group_by { |j| class_name_for_job(j) } + .transform_values(&:shuffle) end private diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index f38d847b0e8..c27ae6a2c5d 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -4,6 +4,7 @@ module Gitlab module Database module Migrations class TestBatchedBackgroundRunner < BaseBackgroundRunner + include Gitlab::Database::DynamicModelHelpers attr_reader :connection def initialize(result_dir:, connection:) @@ -18,31 +19,81 @@ module Gitlab .to_h do |migration| batching_strategy = migration.batch_class.new(connection: connection) - all_migration_jobs = [] + smallest_batch_start = migration.next_min_value - min_value = migration.next_min_value + table_max_value = define_batchable_model(migration.table_name, connection: connection) + .maximum(migration.column_name) - while (next_bounds = batching_strategy.next_batch( - migration.table_name, - migration.column_name, - batch_min_value: min_value, - batch_size: migration.batch_size, - job_arguments: migration.job_arguments - )) + largest_batch_start = table_max_value - migration.batch_size + + # variance is the portion of the batch range that we shrink between variance * 0 and variance * 1 + # to pick actual batches to sample. + variance = largest_batch_start - smallest_batch_start + + batch_starts = uniform_fractions + .lazy # frac varies from 0 to 1, values in smallest_batch_start..largest_batch_start + .map { |frac| (variance * frac).to_i + smallest_batch_start } + + # Track previously run batches so that we stop sampling if a new batch would intersect an older one + completed_batches = [] + + jobs_to_sample = batch_starts + # Stop sampling if a batch would intersect a previous batch + .take_while { |start| completed_batches.none? { |batch| batch.cover?(start) } } + .map do |batch_start| + next_bounds = batching_strategy.next_batch( + migration.table_name, + migration.column_name, + batch_min_value: batch_start, + batch_size: migration.batch_size, + job_arguments: migration.job_arguments + ) batch_min, batch_max = next_bounds - all_migration_jobs << migration.create_batched_job!(batch_min, batch_max) - min_value = batch_max + 1 + job = migration.create_batched_job!(batch_min, batch_max) + + completed_batches << (batch_min..batch_max) + + job end - [migration.job_class_name, all_migration_jobs] + [migration.job_class_name, jobs_to_sample] end end def run_job(job) Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper.new(connection: connection).perform(job) end + + def uniform_fractions + Enumerator.new do |y| + # Generates equally distributed fractions between 0 and 1, with increasing detail as more are pulled from + # the enumerator. + # 0, 1 (special case) + # 1/2 + # 1/4, 3/4 + # 1/8, 3/8, 5/8, 7/8 + # etc. + # The pattern here is at each outer loop, the denominator multiplies by 2, and at each inner loop, + # the numerator counts up all odd numbers 1 <= n < denominator. + y << 0 + y << 1 + + # denominators are each increasing power of 2 + denominators = (1..).lazy.map { |exponent| 2**exponent } + + denominators.each do |denominator| + # Numerators at the current step are all odd numbers between 1 and the denominator + numerators = (1..denominator).step(2) + + numerators.each do |numerator| + next_frac = numerator.fdiv(denominator) + y << next_frac + end + end + end + end end end end diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 92825d41599..6314aff9914 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -33,6 +33,18 @@ module Gitlab PartitionManager.new(model).sync_partitions end + unless only_on + models_to_sync.each do |model| + next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) + + Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + if connection_name != model.connection_db_config.name + PartitionManager.new(model, connection: connection).sync_partitions + end + end + end + end + Gitlab::AppLogger.info(message: 'Finished sync of dynamic postgres partitions') end diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb new file mode 100644 index 00000000000..f45cf02ec9b --- /dev/null +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -0,0 +1,214 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Partitioning + class ConvertTableToFirstListPartition + UnableToPartition = Class.new(StandardError) + + include Gitlab::Database::MigrationHelpers + + SQL_STATEMENT_SEPARATOR = ";\n\n" + + attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value + + def initialize(migration_context:, table_name:, parent_table_name:, partitioning_column:, zero_partition_value:) + @migration_context = migration_context + @connection = migration_context.connection + @table_name = table_name + @parent_table_name = parent_table_name + @partitioning_column = partitioning_column + @zero_partition_value = zero_partition_value + end + + def prepare_for_partitioning + assert_existing_constraints_partitionable + + add_partitioning_check_constraint + end + + def revert_preparation_for_partitioning + migration_context.remove_check_constraint(table_name, partitioning_constraint.name) + end + + def partition + assert_existing_constraints_partitionable + assert_partitioning_constraint_present + create_parent_table + attach_foreign_keys_to_parent + + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.execute(sql_to_convert_table) + end + end + + def revert_partitioning + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.execute(<<~SQL) + ALTER TABLE #{connection.quote_table_name(parent_table_name)} + DETACH PARTITION #{connection.quote_table_name(table_name)}; + SQL + + alter_sequences_sql = alter_sequence_statements(old_table: parent_table_name, new_table: table_name) + .join(SQL_STATEMENT_SEPARATOR) + + migration_context.execute(alter_sequences_sql) + + # This takes locks for all the foreign keys that the parent table had. + # However, those same locks were taken while detaching the partition, and we can't avoid that. + # If we dropped the foreign key before detaching the partition to avoid this locking, + # the drop would cascade to the child partitions and drop their foreign keys as well + migration_context.drop_table(parent_table_name) + end + + add_partitioning_check_constraint + end + + private + + attr_reader :connection, :migration_context + + delegate :quote_table_name, :quote_column_name, to: :connection + + def sql_to_convert_table + # The critical statement here is the attach_table_to_parent statement. + # The following statements could be run in a later transaction, + # but they acquire the same locks so it's much faster to incude them + # here. + [ + attach_table_to_parent_statement, + alter_sequence_statements(old_table: table_name, new_table: parent_table_name), + remove_constraint_statement + ].flatten.join(SQL_STATEMENT_SEPARATOR) + end + + def table_identifier + "#{connection.current_schema}.#{table_name}" + end + + def assert_existing_constraints_partitionable + violating_constraints = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .primary_or_unique_constraints + .not_including_column(partitioning_column) + .to_a + + return if violating_constraints.empty? + + violation_messages = violating_constraints.map { |c| "#{c.name} on (#{c.column_names.join(', ')})" } + + raise UnableToPartition, <<~MSG + Constraints on #{table_name} are incompatible with partitioning on #{partitioning_column} + + All primary key and unique constraints must include the partitioning column. + Violations: + #{violation_messages.join("\n")} + MSG + end + + def partitioning_constraint + constraints_on_column = Gitlab::Database::PostgresConstraint + .by_table_identifier(table_identifier) + .check_constraints + .valid + .including_column(partitioning_column) + + constraints_on_column.to_a.find do |constraint| + constraint.definition == "CHECK ((#{partitioning_column} = #{zero_partition_value}))" + end + end + + def assert_partitioning_constraint_present + return if partitioning_constraint + + raise UnableToPartition, <<~MSG + Table #{table_name} is not ready for partitioning. + Before partitioning, a check constraint must enforce that (#{partitioning_column} = #{zero_partition_value}) + MSG + end + + def add_partitioning_check_constraint + return if partitioning_constraint.present? + + check_body = "#{partitioning_column} = #{connection.quote(zero_partition_value)}" + # Any constraint name would work. The constraint is found based on its definition before partitioning + migration_context.add_check_constraint(table_name, check_body, 'partitioning_constraint') + + raise UnableToPartition, 'Error adding partitioning constraint' unless partitioning_constraint.present? + end + + def create_parent_table + migration_context.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS #{quote_table_name(parent_table_name)} ( + LIKE #{quote_table_name(table_name)} INCLUDING ALL + ) PARTITION BY LIST(#{quote_column_name(partitioning_column)}) + SQL + end + + def attach_foreign_keys_to_parent + migration_context.foreign_keys(table_name).each do |fk| + # At this point no other connection knows about the parent table. + # Thus the only contended lock in the following transaction is on fk.to_table. + # So a deadlock is impossible. + + # If we're rerunning this migration after a failure to acquire a lock, the foreign key might already exist. + # Don't try to recreate it in that case + if migration_context.foreign_keys(parent_table_name) + .any? { |p_fk| p_fk.options[:name] == fk.options[:name] } + next + end + + migration_context.with_lock_retries(raise_on_exhaustion: true) do + migration_context.add_foreign_key(parent_table_name, fk.to_table, **fk.options) + end + end + end + + def attach_table_to_parent_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + ATTACH PARTITION #{table_name} + FOR VALUES IN (#{zero_partition_value}) + SQL + end + + def alter_sequence_statements(old_table:, new_table:) + sequences_owned_by(old_table).map do |seq_info| + seq_name, column_name = seq_info.values_at(:name, :column_name) + <<~SQL.chomp + ALTER SEQUENCE #{quote_table_name(seq_name)} OWNED BY #{quote_table_name(new_table)}.#{quote_column_name(column_name)} + SQL + end + end + + def remove_constraint_statement + <<~SQL + ALTER TABLE #{quote_table_name(parent_table_name)} + DROP CONSTRAINT #{quote_table_name(partitioning_constraint.name)} + SQL + end + + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/373887 + def sequences_owned_by(table_name) + sequence_data = connection.exec_query(<<~SQL, nil, [table_name]) + SELECT seq_pg_class.relname AS seq_name, + dep_pg_class.relname AS table_name, + pg_attribute.attname AS col_name + FROM pg_class seq_pg_class + INNER JOIN pg_depend ON seq_pg_class.oid = pg_depend.objid + INNER JOIN pg_class dep_pg_class ON pg_depend.refobjid = dep_pg_class.oid + INNER JOIN pg_attribute ON dep_pg_class.oid = pg_attribute.attrelid + AND pg_depend.refobjsubid = pg_attribute.attnum + WHERE seq_pg_class.relkind = 'S' + AND dep_pg_class.relname = $1 + SQL + + sequence_data.map do |seq_info| + name, column_name = seq_info.values_at('seq_name', 'col_name') + { name: name, column_name: column_name } + end + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index aac91eaadb1..55ca9ff8645 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -10,12 +10,15 @@ module Gitlab MANAGEMENT_LEASE_KEY = 'database_partition_management_%s' RETAIN_DETACHED_PARTITIONS_FOR = 1.week - def initialize(model) + def initialize(model, connection: nil) @model = model - @connection_name = model.connection.pool.db_config.name + @connection = connection || model.connection + @connection_name = @connection.pool.db_config.name end def sync_partitions + return skip_synching_partitions unless table_partitioned? + Gitlab::AppLogger.info( message: "Checking state of dynamic postgres partitions", table_name: model.table_name, @@ -43,9 +46,7 @@ module Gitlab private - attr_reader :model - - delegate :connection, to: :model + attr_reader :model, :connection def missing_partitions return [] unless connection.table_exists?(model.table_name) @@ -129,6 +130,20 @@ module Gitlab connection: connection ).run(&block) end + + def table_partitioned? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(model.table_name).present? + end + end + + def skip_synching_partitions + Gitlab::AppLogger.warn( + message: "Skipping synching partitions", + table_name: model.table_name, + connection_name: @connection_name + ) + end end end end diff --git a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb index 23ac73a0e53..4e38eea963b 100644 --- a/lib/gitlab/database/partitioning/single_numeric_list_partition.rb +++ b/lib/gitlab/database/partitioning/single_numeric_list_partition.rb @@ -8,7 +8,7 @@ module Gitlab def self.from_sql(table, partition_name, definition) # A list partition can support multiple values, but we only support a single number - matches = definition.match(/FOR VALUES IN \('(?<value>\d+)'\)/) + matches = definition.match(/FOR VALUES IN \('?(?<value>\d+)'?\)/) raise ArgumentError, 'Unknown partition definition' unless matches @@ -29,17 +29,21 @@ module Gitlab @partition_name || "#{table}_#{value}" end + def data_size + execute("SELECT pg_table_size(#{quote(full_partition_name)})").first['pg_table_size'] + end + def to_sql <<~SQL CREATE TABLE IF NOT EXISTS #{fully_qualified_partition} - PARTITION OF #{conn.quote_table_name(table)} - FOR VALUES IN (#{conn.quote(value)}) + PARTITION OF #{quote_table_name(table)} + FOR VALUES IN (#{quote(value)}) SQL end def to_detach_sql <<~SQL - ALTER TABLE #{conn.quote_table_name(table)} + ALTER TABLE #{quote_table_name(table)} DETACH PARTITION #{fully_qualified_partition} SQL end @@ -63,8 +67,14 @@ module Gitlab private + delegate :execute, :quote, :quote_table_name, to: :conn, private: true + + def full_partition_name + "%s.%s" % [Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA, partition_name] + end + def fully_qualified_partition - "%s.%s" % [conn.quote_table_name(Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA), conn.quote_table_name(partition_name)] + quote_table_name(full_partition_name) end def conn diff --git a/lib/gitlab/database/partitioning/sliding_list_strategy.rb b/lib/gitlab/database/partitioning/sliding_list_strategy.rb index 4b5349f0327..5bb34a86d43 100644 --- a/lib/gitlab/database/partitioning/sliding_list_strategy.rb +++ b/lib/gitlab/database/partitioning/sliding_list_strategy.rb @@ -14,7 +14,7 @@ module Gitlab @next_partition_if = next_partition_if @detach_partition_if = detach_partition_if - ensure_partitioning_column_ignored! + ensure_partitioning_column_ignored_or_readonly! end def current_partitions @@ -26,7 +26,7 @@ module Gitlab def missing_partitions if no_partitions_exist? [initial_partition] - elsif next_partition_if.call(active_partition.value) + elsif next_partition_if.call(active_partition) [next_partition] else [] @@ -44,7 +44,7 @@ module Gitlab def extra_partitions possibly_extra = current_partitions[0...-1] # Never consider the most recent partition - extra = possibly_extra.take_while { |p| detach_partition_if.call(p.value) } + extra = possibly_extra.take_while { |p| detach_partition_if.call(p) } default_value = current_default_value if extra.any? { |p| p.value == default_value } @@ -128,12 +128,17 @@ module Gitlab Integer(value) end - def ensure_partitioning_column_ignored! - unless model.ignored_columns.include?(partitioning_key.to_s) - raise "Add #{partitioning_key} to #{model.name}.ignored_columns to use it with SlidingListStrategy" + def ensure_partitioning_column_ignored_or_readonly! + unless key_ignored_or_readonly? + raise "Add #{partitioning_key} to #{model.name}.ignored_columns or " \ + "mark it as readonly to use it with SlidingListStrategy" end end + def key_ignored_or_readonly? + model.ignored_columns.include?(partitioning_key.to_s) || model.readonly_attribute?(partitioning_key.to_s) + end + def with_lock_retries(&block) Gitlab::Database::WithLockRetries.new( klass: self.class, diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index c9a3b5caf79..15b542cf089 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -77,8 +77,42 @@ module Gitlab end end + # Finds duplicate indexes for a given schema and table. This finds + # indexes where the index definition is identical but the names are + # different. Returns an array of arrays containing duplicate index name + # pairs. + # + # Example: + # + # find_duplicate_indexes('table_name_goes_here') + def find_duplicate_indexes(table_name, schema_name: connection.current_schema) + find_indexes(table_name, schema_name: schema_name) + .group_by { |r| r['index_id'] } + .select { |_, v| v.size > 1 } + .map { |_, indexes| indexes.map { |index| index['index_name'] } } + end + private + def find_indexes(table_name, schema_name: connection.current_schema) + indexes = connection.select_all(<<~SQL, 'SQL', [schema_name, table_name]) + SELECT n.nspname AS schema_name, + c.relname AS table_name, + i.relname AS index_name, + regexp_replace(pg_get_indexdef(i.oid), 'INDEX .*? USING', '_') AS index_id + FROM pg_index x + JOIN pg_class c ON c.oid = x.indrelid + JOIN pg_class i ON i.oid = x.indexrelid + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE (c.relkind = ANY (ARRAY['r'::"char", 'm'::"char", 'p'::"char"])) + AND (i.relkind = ANY (ARRAY['i'::"char", 'I'::"char"])) + AND n.nspname = $1 + AND c.relname = $2; + SQL + + indexes.to_a + end + def find_partitioned_table(table_name) partitioned_table = Gitlab::Database::PostgresPartitionedTable.find_by_name_in_current_schema(table_name) diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index a541ecf5316..695a5d7ec77 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -251,6 +251,54 @@ module Gitlab create_sync_trigger(source_table_name, trigger_name, function_name) end + def prepare_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:prepare_constraint_for_list_partitioning) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).prepare_for_partitioning + end + + def revert_preparing_constraint_for_list_partitioning(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:revert_preparing_constraint_for_list_partitioning) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).revert_preparation_for_partitioning + end + + def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:convert_table_to_first_list_partition) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).partition + end + + def revert_converting_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + validate_not_in_transaction!(:revert_converting_table_to_first_list_partition) + + Gitlab::Database::Partitioning::ConvertTableToFirstListPartition + .new(migration_context: self, + table_name: table_name, + parent_table_name: parent_table_name, + partitioning_column: partitioning_column, + zero_partition_value: initial_partitioning_value + ).revert_partitioning + end + private def assert_table_is_allowed(table_name) diff --git a/lib/gitlab/database/postgres_constraint.rb b/lib/gitlab/database/postgres_constraint.rb new file mode 100644 index 00000000000..fa590914332 --- /dev/null +++ b/lib/gitlab/database/postgres_constraint.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # Backed by the postgres_constraints view + class PostgresConstraint < SharedModel + IDENTIFIER_REGEX = /^\w+\.\w+$/.freeze + self.primary_key = :oid + + scope :check_constraints, -> { where(constraint_type: 'c') } + scope :primary_key_constraints, -> { where(constraint_type: 'p') } + scope :unique_constraints, -> { where(constraint_type: 'u') } + scope :primary_or_unique_constraints, -> { where(constraint_type: %w[u p]) } + + scope :including_column, ->(column) { where("? = ANY(column_names)", column) } + scope :not_including_column, ->(column) { where.not("? = ANY(column_names)", column) } + + scope :valid, -> { where(constraint_valid: true) } + + scope :by_table_identifier, ->(identifier) do + unless identifier =~ IDENTIFIER_REGEX + raise ArgumentError, "Table name is not fully qualified with a schema: #{identifier}" + end + + where(table_identifier: identifier) + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb new file mode 100644 index 00000000000..c2d5dfc1a15 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + module Ci + # The purpose of this analyzer is to detect queries not going through a partitioning routing table + class PartitioningAnalyzer < Database::QueryAnalyzers::Base + RoutingTableNotUsedError = Class.new(QueryAnalyzerError) + + ENABLED_TABLES = %w[ + ci_builds_metadata + ].freeze + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + ::Feature.enabled?(:ci_partitioning_analyze_queries, type: :ops) + end + + def analyze(parsed) + analyze_legacy_tables_usage(parsed) + end + + private + + def analyze_legacy_tables_usage(parsed) + detected = ENABLED_TABLES & (parsed.pg.dml_tables + parsed.pg.select_tables) + + return if detected.none? + + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + RoutingTableNotUsedError.new("Detected non-partitioned table use #{detected.inspect}: #{parsed.sql}") + ) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb index 06e2b114c91..b4b9161f0c2 100644 --- a/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb +++ b/lib/gitlab/database/query_analyzers/gitlab_schemas_metrics.rb @@ -14,7 +14,7 @@ module Gitlab class << self def enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:query_analyzer_gitlab_schema_metrics) + Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops) end def analyze(parsed) diff --git a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb index e0cb803b872..3b1751c863d 100644 --- a/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb +++ b/lib/gitlab/database/query_analyzers/prevent_cross_database_modification.rb @@ -33,7 +33,7 @@ module Gitlab def self.enabled? ::Feature::FlipperFeature.table_exists? && - Feature.enabled?(:detect_cross_database_modification) + Feature.enabled?(:detect_cross_database_modification, type: :ops) end def self.requires_tracking?(parsed) diff --git a/lib/gitlab/database/reflection.rb b/lib/gitlab/database/reflection.rb index 3ea7277571f..33c965cb150 100644 --- a/lib/gitlab/database/reflection.rb +++ b/lib/gitlab/database/reflection.rb @@ -114,7 +114,7 @@ module Gitlab 'PostgreSQL on Amazon RDS' => { statement: 'SHOW rds.extensions', error: /PG::UndefinedObject/ }, # Based on https://cloud.google.com/sql/docs/postgres/flags#postgres-c this should be specific # to Cloud SQL for PostgreSQL - 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, + 'Cloud SQL for PostgreSQL' => { statement: 'SHOW cloudsql.iam_authentication', error: /PG::UndefinedObject/ }, # Based on # - https://docs.microsoft.com/en-us/azure/postgresql/flexible-server/concepts-extensions # - https://docs.microsoft.com/en-us/azure/postgresql/concepts-extensions diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index b96dffc99ac..aba45fcc57b 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -27,7 +27,7 @@ module Gitlab # Hack: Before we do actual reindexing work, create async indexes Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops) - Gitlab::Database::AsyncIndexes.drop_pending_indexes! if Feature.enabled?(:database_async_index_destruction, type: :ops) + Gitlab::Database::AsyncIndexes.drop_pending_indexes! automatic_reindexing end diff --git a/lib/gitlab/database/tables_sorted_by_foreign_keys.rb b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb new file mode 100644 index 00000000000..9f096904d31 --- /dev/null +++ b/lib/gitlab/database/tables_sorted_by_foreign_keys.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesSortedByForeignKeys + include TSort + + def initialize(connection, tables) + @connection = connection + @tables = tables + end + + def execute + strongly_connected_components + end + + private + + def tsort_each_node(&block) + tables_dependencies.each_key(&block) + end + + def tsort_each_child(node, &block) + tables_dependencies[node].each(&block) + end + + # it maps the tables to the tables that depend on it + def tables_dependencies + @tables.to_h do |table_name| + [table_name, all_foreign_keys[table_name]&.map(&:from_table).to_a] + end + end + + def all_foreign_keys + @all_foreign_keys ||= @tables.flat_map do |table_name| + @connection.foreign_keys(table_name) + end.group_by(&:to_table) + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb new file mode 100644 index 00000000000..164520fbab3 --- /dev/null +++ b/lib/gitlab/database/tables_truncate.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class TablesTruncate + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + + def initialize(database_name:, min_batch_size:, logger: nil, until_table: nil, dry_run: false) + @database_name = database_name + @min_batch_size = min_batch_size + @logger = logger + @until_table = until_table + @dry_run = dry_run + end + + def execute + raise "Cannot truncate legacy tables in single-db setup" unless Gitlab::Database.has_config?(:ci) + raise "database is not supported" unless %w[main ci].include?(database_name) + + logger&.info "DRY RUN:" if dry_run + + connection = Gitlab::Database.database_base_models[database_name].connection + + schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) + tables_to_truncate = Gitlab::Database::GitlabSchema.tables_to_schema.reject do |_, schema_name| + (GITLAB_SCHEMAS_TO_IGNORE.union(schemas_for_connection)).include?(schema_name) + end.keys + + tables_sorted = Gitlab::Database::TablesSortedByForeignKeys.new(connection, tables_to_truncate).execute + # Checking if all the tables have the write-lock triggers + # to make sure we are deleting the right tables on the right database. + tables_sorted.flatten.each do |table_name| + query = <<~SQL + SELECT COUNT(*) from information_schema.triggers + WHERE event_object_table = '#{table_name}' + AND trigger_name = 'gitlab_schema_write_trigger_for_#{table_name}' + SQL + + if connection.select_value(query) == 0 + raise "Table '#{table_name}' is not locked for writes. Run the rake task gitlab:db:lock_writes first" + end + end + + if until_table + table_index = tables_sorted.find_index { |tables_group| tables_group.include?(until_table) } + raise "The table '#{until_table}' is not within the truncated tables" if table_index.nil? + + tables_sorted = tables_sorted[0..table_index] + end + + # min_batch_size is the minimum number of new tables to truncate at each stage. + # But in each stage we have also have to truncate the already truncated tables in the previous stages + logger&.info "Truncating legacy tables for the database #{database_name}" + truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + end + + private + + attr_accessor :database_name, :min_batch_size, :logger, :dry_run, :until_table + + def truncate_tables_in_batches(connection, tables_sorted, min_batch_size) + truncated_tables = [] + + tables_sorted.flatten.each do |table| + sql_statement = "SELECT set_config('lock_writes.#{table}', 'false', false)" + logger&.info(sql_statement) + connection.execute(sql_statement) unless dry_run + end + + # We do the truncation in stages to avoid high IO + # In each stage, we truncate the new tables along with the already truncated + # tables before. That's because PostgreSQL doesn't allow to truncate any table (A) + # without truncating any other table (B) that has a Foreign Key pointing to the table (A). + # even if table (B) is empty, because it has been already truncated in a previous stage. + tables_sorted.in_groups_of(min_batch_size, false).each do |tables_groups| + new_tables_to_truncate = tables_groups.flatten + logger&.info "= New tables to truncate: #{new_tables_to_truncate.join(', ')}" + truncated_tables.push(*new_tables_to_truncate).tap(&:sort!) + sql_statements = [ + "SET LOCAL statement_timeout = 0", + "SET LOCAL lock_timeout = 0", + "TRUNCATE TABLE #{truncated_tables.join(', ')} RESTRICT" + ] + + sql_statements.each { |sql_statement| logger&.info(sql_statement) } + + next if dry_run + + connection.transaction do + sql_statements.each { |sql_statement| connection.execute(sql_statement) } + end + end + end + end + end +end diff --git a/lib/gitlab/database_importers/security/training_providers/importer.rb b/lib/gitlab/database_importers/security/training_providers/importer.rb new file mode 100644 index 00000000000..aa6a9f29c6d --- /dev/null +++ b/lib/gitlab/database_importers/security/training_providers/importer.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module DatabaseImporters + module Security + module TrainingProviders + module Importer + KONTRA_DATA = { + name: 'Kontra', + description: "Kontra Application Security provides interactive developer security education that + enables engineers to quickly learn security best practices + and fix issues in their code by analysing real-world software security vulnerabilities.", + url: "https://application.security/api/webhook/gitlab/exercises/search" + }.freeze + + SCW_DATA = { + name: 'Secure Code Warrior', + description: "Resolve vulnerabilities faster and confidently with + highly relevant and bite-sized secure coding learning.", + url: "https://integration-api.securecodewarrior.com/api/v1/trial" + }.freeze + + module Security + class TrainingProvider < ApplicationRecord + self.table_name = 'security_training_providers' + end + end + + def self.upsert_providers + current_time = Time.current + timestamps = { created_at: current_time, updated_at: current_time } + + Security::TrainingProvider.upsert_all( + [KONTRA_DATA.merge(timestamps), SCW_DATA.merge(timestamps)], + unique_by: :index_security_training_providers_on_unique_name + ) + end + end + end + end + end +end diff --git a/lib/gitlab/diff/file_collection/compare.rb b/lib/gitlab/diff/file_collection/compare.rb index badebabb192..6d8395d048d 100644 --- a/lib/gitlab/diff/file_collection/compare.rb +++ b/lib/gitlab/diff/file_collection/compare.rb @@ -6,9 +6,9 @@ module Gitlab class Compare < Base def initialize(compare, project:, diff_options:, diff_refs: nil) super(compare, - project: project, + project: project, diff_options: diff_options, - diff_refs: diff_refs) + diff_refs: diff_refs) end def unfold_diff_lines(positions) diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index 7cfe0086f57..084ce63e36a 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -6,7 +6,7 @@ module Gitlab include Gitlab::Utils::Gzip include Gitlab::Utils::StrongMemoize - EXPIRATION = 1.day + EXPIRATION = 1.hour VERSION = 2 delegate :diffable, to: :@diff_collection @@ -82,6 +82,16 @@ module Gitlab private + def expiration + return 1.day unless Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) + + if Feature.enabled?(:highlight_diffs_short_renewable_expiration, diffable.project) + EXPIRATION + else + 8.hours + end + end + def set_highlighted_diff_lines(diff_file, content) diff_file.highlighted_diff_lines = content.map do |line| Gitlab::Diff::Line.safe_init_from_hash(line) @@ -125,9 +135,9 @@ module Gitlab # def write_to_redis_hash(hash) Gitlab::Redis::Cache.with do |redis| - redis.pipelined do + redis.pipelined do |pipeline| hash.each do |diff_file_id, highlighted_diff_lines_hash| - redis.hset( + pipeline.hset( key, diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) @@ -137,8 +147,7 @@ module Gitlab end # HSETs have to have their expiration date manually updated - # - redis.expire(key, EXPIRATION) + pipeline.expire(key, expiration) end record_memory_usage(fetch_memory_usage(redis, key)) @@ -188,11 +197,19 @@ module Gitlab return {} unless file_paths.any? results = [] + cache_key = key + highlight_diffs_renewable_expiration_enabled = Feature.enabled?(:highlight_diffs_renewable_expiration, diffable.project) + expiration_period = expiration Gitlab::Redis::Cache.with do |redis| - results = redis.hmget(key, file_paths) + redis.pipelined do |pipeline| + results = pipeline.hmget(cache_key, file_paths) + pipeline.expire(key, expiration_period) if highlight_diffs_renewable_expiration_enabled + end end + results = results.value + record_hit_ratio(results) results.map! do |result| diff --git a/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb deleted file mode 100644 index 4bfb5f9e64c..00000000000 --- a/lib/gitlab/doorkeeper_secret_storing/pbkdf2_sha512.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module DoorkeeperSecretStoring - class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base - STRETCHES = 20_000 - # An empty salt is used because we need to look tokens up solely by - # their hashed value. Additionally, tokens are always cryptographically - # pseudo-random and unique, therefore salting provides no - # additional security. - SALT = '' - - def self.transform_secret(plain_secret) - return plain_secret unless Feature.enabled?(:hash_oauth_tokens) - - Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) - end - - ## - # Determines whether this strategy supports restoring - # secrets from the database. This allows detecting users - # trying to use a non-restorable strategy with +reuse_access_tokens+. - def self.allows_restoring_secrets? - false - end - end - end -end diff --git a/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb new file mode 100644 index 00000000000..e0884557496 --- /dev/null +++ b/lib/gitlab/doorkeeper_secret_storing/secret/pbkdf2_sha512.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +module Gitlab + module DoorkeeperSecretStoring + module Secret + class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base + STRETCHES = 20_000 + # An empty salt is used because we need to look tokens up solely by + # their hashed value. Additionally, tokens are always cryptographically + # pseudo-random and unique, therefore salting provides no + # additional security. + SALT = '' + + def self.transform_secret(plain_secret, stored_as_hash = false) + return plain_secret if Feature.disabled?(:hash_oauth_secrets) && !stored_as_hash + + Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) + end + + ## + # Determines whether this strategy supports restoring + # secrets from the database. This allows detecting users + # trying to use a non-restorable strategy with +reuse_access_tokens+. + def self.allows_restoring_secrets? + false + end + + ## + # Securely compare the given +input+ value with a +stored+ value + # processed by +transform_secret+. + def self.secret_matches?(input, stored) + stored_as_hash = stored.starts_with?('$pbkdf2-') + transformed_input = transform_secret(input, stored_as_hash) + ActiveSupport::SecurityUtils.secure_compare transformed_input, stored + end + end + end + end +end diff --git a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb new file mode 100644 index 00000000000..f9e6d4076f3 --- /dev/null +++ b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module DoorkeeperSecretStoring + module Token + class Pbkdf2Sha512 < ::Doorkeeper::SecretStoring::Base + STRETCHES = 20_000 + # An empty salt is used because we need to look tokens up solely by + # their hashed value. Additionally, tokens are always cryptographically + # pseudo-random and unique, therefore salting provides no + # additional security. + SALT = '' + + def self.transform_secret(plain_secret) + return plain_secret unless Feature.enabled?(:hash_oauth_tokens) + + Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) + end + + ## + # Determines whether this strategy supports restoring + # secrets from the database. This allows detecting users + # trying to use a non-restorable strategy with +reuse_access_tokens+. + def self.allows_restoring_secrets? + false + end + end + end + end +end diff --git a/lib/gitlab/email/attachment_uploader.rb b/lib/gitlab/email/attachment_uploader.rb index b67ca8d8a7d..931276588f0 100644 --- a/lib/gitlab/email/attachment_uploader.rb +++ b/lib/gitlab/email/attachment_uploader.rb @@ -20,8 +20,8 @@ module Gitlab sanitize_exif_if_needed(content, tmp.path) file = { - tempfile: tmp, - filename: attachment.filename, + tempfile: tmp, + filename: attachment.filename, content_type: attachment.content_type } diff --git a/lib/gitlab/email/message/in_product_marketing/team.rb b/lib/gitlab/email/message/in_product_marketing/team.rb index 6a0471ef9c5..ca99dd12c8e 100644 --- a/lib/gitlab/email/message/in_product_marketing/team.rb +++ b/lib/gitlab/email/message/in_product_marketing/team.rb @@ -42,18 +42,18 @@ module Gitlab [ s_('InProductMarketing|Did you know teams that use GitLab are far more efficient?'), list([ - s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), - s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') - ]) + s_('InProductMarketing|Goldman Sachs went from 1 build every two weeks to thousands of builds a day'), + s_('InProductMarketing|Ticketmaster decreased their CI build time by 15X') + ]) ].join("\n"), s_("InProductMarketing|We know a thing or two about efficiency and we don't want to keep that to ourselves. Sign up for a free trial of GitLab Ultimate and your teams will be on it from day one."), [ s_('InProductMarketing|Stop wondering and use GitLab to answer questions like:'), list([ - s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), - s_('InProductMarketing|How many days does it take our team to complete various tasks?'), - s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') - ]) + s_('InProductMarketing|How long does it take us to close issues/MRs by types like feature requests, bugs, tech debt, security?'), + s_('InProductMarketing|How many days does it take our team to complete various tasks?'), + s_('InProductMarketing|What does our value stream timeline look like from product to development to review and production?') + ]) ].join("\n") ][series] end diff --git a/lib/gitlab/email/message/repository_push.rb b/lib/gitlab/email/message/repository_push.rb index d55cf3202a6..293aa3b53bf 100644 --- a/lib/gitlab/email/message/repository_push.rb +++ b/lib/gitlab/email/message/repository_push.rb @@ -79,11 +79,11 @@ module Gitlab @action_name ||= case @action when :create - 'pushed new' + s_('Notify|pushed new') when :delete - 'deleted' + s_('Notify|deleted') else - 'pushed to' + s_('Notify|pushed to') end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index f539d627dcb..2b36b1c99bd 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -17,13 +17,13 @@ module Gitlab def emoji_image_tag(name, src) image_options = { - class: 'emoji', - src: src, - title: ":#{name}:", - alt: ":#{name}:", + class: 'emoji', + src: src, + title: ":#{name}:", + alt: ":#{name}:", height: 20, - width: 20, - align: 'absmiddle' + width: 20, + align: 'absmiddle' } ActionController::Base.helpers.tag(:img, image_options) diff --git a/lib/gitlab/encoding_helper.rb b/lib/gitlab/encoding_helper.rb index f26ab6e3ed1..34c674c3003 100644 --- a/lib/gitlab/encoding_helper.rb +++ b/lib/gitlab/encoding_helper.rb @@ -71,6 +71,21 @@ module Gitlab encode_utf8(data, replace: UNICODE_REPLACEMENT_CHARACTER) end + # This method escapes unsupported UTF-8 characters instead of deleting them + def encode_utf8_with_escaping!(message) + return encode!(message) if Feature.disabled?(:escape_gitaly_refs) + + message = force_encode_utf8(message) + return message if message.valid_encoding? + + unless message.valid_encoding? + message = message.chars.map { |char| char.valid_encoding? ? char : escape_chars(char) }.join + end + + # encode and clean the bad chars + message.replace clean(message) + end + def encode_utf8(message, replace: "") message = force_encode_utf8(message) return message if message.valid_encoding? @@ -145,6 +160,15 @@ module Gitlab message.force_encoding("UTF-8") end + # Escapes \x80 - \xFF characters not supported by UTF-8 + def escape_chars(char) + bytes = char.bytes + + return char unless bytes.one? + + "%#{bytes.first.to_s(16).upcase}" + end + def clean(message, replace: "") message.encode( "UTF-16BE", diff --git a/lib/gitlab/etag_caching/middleware.rb b/lib/gitlab/etag_caching/middleware.rb index a1918ee6ad5..f6431483a15 100644 --- a/lib/gitlab/etag_caching/middleware.rb +++ b/lib/gitlab/etag_caching/middleware.rb @@ -97,12 +97,12 @@ module Gitlab def add_instrument_for_cache_hit(status, route, request) payload = { etag_route: route.name, - params: request.filtered_parameters, - headers: request.headers, - format: request.format.ref, - method: request.request_method, - path: request.filtered_path, - status: status + params: request.filtered_parameters, + headers: request.headers, + format: request.format.ref, + method: request.request_method, + path: request.filtered_path, + status: status } ActiveSupport::Notifications.instrument( diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 44c6984c09b..437d577e70e 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -16,9 +16,9 @@ module Gitlab etags = keys.map { generate_etag } Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do + redis.pipelined do |pipeline| keys.each_with_index do |key, i| - redis.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) end end end diff --git a/lib/gitlab/experimentation.rb b/lib/gitlab/experimentation.rb index 8a5432025d8..142d0e55593 100644 --- a/lib/gitlab/experimentation.rb +++ b/lib/gitlab/experimentation.rb @@ -70,9 +70,9 @@ module Gitlab logger = Gitlab::ExperimentationLogger.build logger.warn message: 'Subject must conform to the rollout strategy', - experiment_key: experiment_key, - subject: subject.class.to_s, - rollout_strategy: rollout_strategy(experiment_key) + experiment_key: experiment_key, + subject: subject.class.to_s, + rollout_strategy: rollout_strategy(experiment_key) end def valid_subject_for_rollout_strategy?(experiment_key, subject) diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index 509daeb0248..c06711d16f8 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -20,8 +20,8 @@ module Gitlab def store(new_access, new_reason, new_refreshed_at) ::Gitlab::Redis::Cache.with do |redis| - redis.pipelined do - redis.mapped_hmset( + redis.pipelined do |pipeline| + pipeline.mapped_hmset( cache_key, { access: new_access.to_s, @@ -30,7 +30,7 @@ module Gitlab } ) - redis.expire(cache_key, VALIDITY_TIME) + pipeline.expire(cache_key, VALIDITY_TIME) end end end diff --git a/lib/gitlab/fogbugz_import/importer.rb b/lib/gitlab/fogbugz_import/importer.rb index 612865ed1be..ca1a2b2a077 100644 --- a/lib/gitlab/fogbugz_import/importer.rb +++ b/lib/gitlab/fogbugz_import/importer.rb @@ -129,15 +129,15 @@ module Gitlab author_id = user_info(bug['ixPersonOpenedBy'])[:gitlab_id] || project.creator_id issue = Issue.create!( - iid: bug['ixBug'], - project_id: project.id, - title: bug['sTitle'], - description: body, - author_id: author_id, + iid: bug['ixBug'], + project_id: project.id, + title: bug['sTitle'], + description: body, + author_id: author_id, assignee_ids: [assignee_id], - state: bug['fOpen'] == 'true' ? 'opened' : 'closed', - created_at: date, - updated_at: DateTime.parse(bug['dtLastUpdated']) + state: bug['fOpen'] == 'true' ? 'opened' : 'closed', + created_at: date, + updated_at: DateTime.parse(bug['dtLastUpdated']) ) issue_labels = ::LabelsFinder.new(nil, project_id: project.id, title: labels).execute(skip_authorization: true) @@ -184,11 +184,11 @@ module Gitlab ) note = Note.create!( - project_id: project.id, - noteable_type: "Issue", - noteable_id: issue.id, - author_id: author_id, - note: body + project_id: project.id, + noteable_type: "Issue", + noteable_id: issue.id, + author_id: author_id, + note: body ) note.update_attribute(:created_at, date) diff --git a/lib/gitlab/gfm/reference_rewriter.rb b/lib/gitlab/gfm/reference_rewriter.rb index 40dcac5f46f..0c13ab604bc 100644 --- a/lib/gitlab/gfm/reference_rewriter.rb +++ b/lib/gitlab/gfm/reference_rewriter.rb @@ -55,7 +55,12 @@ module Gitlab end def needs_rewrite? - strong_memoize(:needs_rewrite) { @text_html.include?('data-reference-type=') } + strong_memoize(:needs_rewrite) do + reference_type_attribute = + Banzai::Filter::References::ReferenceFilter::REFERENCE_TYPE_DATA_ATTRIBUTE + + @text_html.include?(reference_type_attribute) + end end private diff --git a/lib/gitlab/git.rb b/lib/gitlab/git.rb index 4b9f2ababc8..4b877bf44da 100644 --- a/lib/gitlab/git.rb +++ b/lib/gitlab/git.rb @@ -25,7 +25,7 @@ module Gitlab include Gitlab::EncodingHelper def ref_name(ref) - encode!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') + encode_utf8_with_escaping!(ref).sub(%r{\Arefs/(tags|heads|remotes)/}, '') end def branch_name(ref) diff --git a/lib/gitlab/git/diff.rb b/lib/gitlab/git/diff.rb index 003cc87d65a..72f7413500f 100644 --- a/lib/gitlab/git/diff.rb +++ b/lib/gitlab/git/diff.rb @@ -230,7 +230,6 @@ module Gitlab private def encode_diff_to_utf8(replace_invalid_utf8_chars) - return unless Feature.enabled?(:convert_diff_to_utf8_with_replacement_symbol) return unless replace_invalid_utf8_chars && diff_should_be_converted? @diff = Gitlab::EncodingHelper.encode_utf8_with_replacement_character(@diff) diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ad655fedb6d..f1cd75258be 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -403,7 +403,7 @@ module Gitlab wrapped_gitaly_errors do gitaly_blob_client.list_blobs(revisions, limit: REV_LIST_COMMIT_LIMIT, - with_paths: with_paths, dynamic_timeout: dynamic_timeout) + with_paths: with_paths, dynamic_timeout: dynamic_timeout) end end @@ -701,7 +701,9 @@ module Gitlab # Delete the specified branch from the repository # Note: No Git hooks are executed for this action def delete_branch(branch_name) - write_ref(branch_name, Gitlab::Git::BLANK_SHA) + branch_name = "#{Gitlab::Git::BRANCH_REF_PREFIX}#{branch_name}" unless branch_name.start_with?("refs/") + + delete_refs(branch_name) rescue CommandError => e raise DeleteBranchError, e end @@ -913,8 +915,29 @@ module Gitlab true end + # Creates a commit + # + # @param [User] user The committer of the commit. + # @param [String] branch_name: The name of the branch to be created/updated. + # @param [String] message: The commit message. + # @param [Array<Hash>] actions: An array of files to be added/updated/removed. + # @option actions: [Symbol] :action One of :create, :create_dir, :update, :move, :delete, :chmod + # @option actions: [String] :file_path The path of the file or directory being added/updated/removed. + # @option actions: [String] :previous_path The path of the file being moved. Only used for the :move action. + # @option actions: [String,IO] :content The file content for :create or :update + # @option actions: [String] :encoding One of text, base64 + # @option actions: [Boolean] :execute_filemode True sets the executable filemode on the file. + # @option actions: [Boolean] :infer_content True uses the existing file contents instead of using content on move. + # @param [String] author_email: The authors email, if unspecified the committers email is used. + # @param [String] author_name: The authors name, if unspecified the committers name is used. + # @param [String] start_branch_name: The name of the branch to be used as the parent of the commit. Only used if start_sha: is unspecified. + # @param [String] start_sha: The sha to be used as the parent of the commit. + # @param [Gitlab::Git::Repository] start_repository: The repository that contains the start branch or sha. Defaults to use this repository. + # @param [Boolean] force: Force update the branch. + # @return [Gitlab::Git::OperationService::BranchUpdate] + # # rubocop:disable Metrics/ParameterLists - def multi_action( + def commit_files( user, branch_name:, message:, actions:, author_email: nil, author_name: nil, start_branch_name: nil, start_sha: nil, start_repository: nil, @@ -989,8 +1012,8 @@ module Gitlab gitaly_ref_client.branch_names_contains_sha(sha) end - def tag_names_contains_sha(sha) - gitaly_ref_client.tag_names_contains_sha(sha) + def tag_names_contains_sha(sha, limit: 0) + gitaly_ref_client.tag_names_contains_sha(sha, limit: limit) end def search_files_by_content(query, ref, options = {}) @@ -1011,16 +1034,20 @@ module Gitlab end def search_files_by_name(query, ref) - safe_query = Regexp.escape(query.sub(%r{^/*}, "")) + safe_query = query.sub(%r{^/*}, "") ref ||= root_ref return [] if empty? || safe_query.blank? - gitaly_repository_client.search_files_by_name(ref, safe_query) + gitaly_repository_client.search_files_by_name(ref, safe_query).map do |file| + Gitlab::EncodingHelper.encode_utf8(file) + end end def search_files_by_regexp(filter, ref = 'HEAD') - gitaly_repository_client.search_files_by_regexp(ref, filter) + gitaly_repository_client.search_files_by_regexp(ref, filter).map do |file| + Gitlab::EncodingHelper.encode_utf8(file) + end end def find_commits_by_message(query, ref, path, limit, offset) @@ -1031,6 +1058,24 @@ module Gitlab end end + def list_commits_by(query, ref, author: nil, before: nil, after: nil, limit: 1000) + params = { + author: author, + ignore_case: true, + commit_message_patterns: query, + before: before, + after: after, + reverse: false, + pagination_params: { limit: limit } + } + + wrapped_gitaly_errors do + gitaly_commit_client + .list_commits([ref], params) + .map { |c| commit(c) } + end + end + def list_last_commits_for_tree(sha, path, offset: 0, limit: 25, literal_pathspec: false) wrapped_gitaly_errors do gitaly_commit_client.list_last_commits_for_tree(sha, path, offset: offset, limit: limit, literal_pathspec: literal_pathspec) diff --git a/lib/gitlab/git/tag.rb b/lib/gitlab/git/tag.rb index 25895dc6728..5ed5158eeea 100644 --- a/lib/gitlab/git/tag.rb +++ b/lib/gitlab/git/tag.rb @@ -63,7 +63,7 @@ module Gitlab end def init_from_gitaly - @name = encode!(@raw_tag.name.dup) + @name = encode_utf8_with_escaping!(@raw_tag.name.dup) @target = @raw_tag.id @message = message_from_gitaly_tag diff --git a/lib/gitlab/git/wiki.rb b/lib/gitlab/git/wiki.rb index 4bab94968d7..2228fcb886e 100644 --- a/lib/gitlab/git/wiki.rb +++ b/lib/gitlab/git/wiki.rb @@ -70,18 +70,6 @@ module Gitlab @repository.exists? end - def write_page(name, format, content, commit_details) - wrapped_gitaly_errors do - gitaly_write_page(name, format, content, commit_details) - end - end - - def update_page(page_path, title, format, content, commit_details) - wrapped_gitaly_errors do - gitaly_update_page(page_path, title, format, content, commit_details) - end - end - def list_pages(limit: 0, sort: nil, direction_desc: false, load_content: false) wrapped_gitaly_errors do gitaly_list_pages( @@ -113,21 +101,13 @@ module Gitlab @gitaly_wiki_client ||= Gitlab::GitalyClient::WikiService.new(@repository) end - def gitaly_write_page(name, format, content, commit_details) - gitaly_wiki_client.write_page(name, format, content, commit_details) - end - - def gitaly_update_page(page_path, title, format, content, commit_details) - gitaly_wiki_client.update_page(page_path, title, format, content, commit_details) - end - def gitaly_find_page(title:, version: nil, dir: nil, load_content: true) return unless title.present? wiki_page, version = gitaly_wiki_client.find_page(title: title, version: version, dir: dir, load_content: load_content) return unless wiki_page - Gitlab::Git::WikiPage.new(wiki_page, version) + Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) rescue GRPC::InvalidArgument nil end @@ -143,7 +123,7 @@ module Gitlab end gitaly_pages.map do |wiki_page, version| - Gitlab::Git::WikiPage.new(wiki_page, version) + Gitlab::Git::WikiPage.from_gitaly_wiki_page(wiki_page, version) end end end diff --git a/lib/gitlab/git/wiki_page.rb b/lib/gitlab/git/wiki_page.rb index a1f3d64ccde..57b7e7d53dd 100644 --- a/lib/gitlab/git/wiki_page.rb +++ b/lib/gitlab/git/wiki_page.rb @@ -5,17 +5,31 @@ module Gitlab class WikiPage attr_reader :url_path, :title, :format, :path, :version, :raw_data, :name, :historical, :formatted_data - # This class abstracts away Gitlab::GitalyClient::WikiPage - def initialize(gitaly_page, version) - @url_path = gitaly_page.url_path - @title = gitaly_page.title - @format = gitaly_page.format - @path = gitaly_page.path - @raw_data = gitaly_page.raw_data - @name = gitaly_page.name - @historical = gitaly_page.historical? + class << self + # Abstracts away Gitlab::GitalyClient::WikiPage + def from_gitaly_wiki_page(gitaly_page, version) + new( + url_path: gitaly_page.url_path, + title: gitaly_page.title, + format: gitaly_page.format, + path: gitaly_page.path, + raw_data: gitaly_page.raw_data, + name: gitaly_page.name, + historical: gitaly_page.historical?, + version: version + ) + end + end - @version = version + def initialize(hash) + @url_path = hash[:url_path] + @title = hash[:title] + @format = hash[:format] + @path = hash[:path] + @raw_data = hash[:raw_data] + @name = hash[:name] + @historical = hash[:historical] + @version = hash[:version] end def historical? diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 0f306a9825d..312d1dddff1 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -232,7 +232,7 @@ module Gitlab msg.paths.map do |path| Gitlab::Git::ChangedPath.new( status: path.status, - path: EncodingHelper.encode!(path.path) + path: EncodingHelper.encode!(path.path) ) end end @@ -251,14 +251,23 @@ module Gitlab consume_commits_response(response) end - def list_commits(revisions, reverse: false, pagination_params: nil) + def list_commits(revisions, params = {}) request = Gitaly::ListCommitsRequest.new( repository: @gitaly_repo, revisions: Array.wrap(revisions), - reverse: reverse, - pagination_params: pagination_params + reverse: !!params[:reverse], + ignore_case: params[:ignore_case], + pagination_params: params[:pagination_params] ) + if params[:commit_message_patterns] + request.commit_message_patterns += Array.wrap(params[:commit_message_patterns]) + end + + request.author = encode_binary(params[:author]) if params[:author] + request.before = GitalyClient.timestamp(params[:before]) if params[:before] + request.after = GitalyClient.timestamp(params[:after]) if params[:after] + response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -396,12 +405,12 @@ module Gitlab def find_commits(options) request = Gitaly::FindCommitsRequest.new( - repository: @gitaly_repo, - limit: options[:limit], - offset: options[:offset], - follow: options[:follow], - skip_merges: options[:skip_merges], - all: !!options[:all], + repository: @gitaly_repo, + limit: options[:limit], + offset: options[:offset], + follow: options[:follow], + skip_merges: options[:skip_merges], + all: !!options[:all], first_parent: !!options[:first_parent], global_options: parse_global_options!(options), disable_walk: true, # This option is deprecated. The 'walk' implementation is being removed. diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index c5c6ec1cdfa..7835fb32f59 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -85,8 +85,20 @@ module Gitlab target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) Gitlab::Git::Branch.new(@repository, branch.name, target_commit.id, target_commit) - rescue GRPC::FailedPrecondition => ex - raise Gitlab::Git::Repository::InvalidRef, ex + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) + else + if e.code == GRPC::Core::StatusCodes::FAILED_PRECONDITION + raise Gitlab::Git::Repository::InvalidRef, e + end + + raise + end end def user_update_branch(branch_name, user, newrev, oldrev) @@ -410,9 +422,9 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, - :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, - remote_storage: start_repository&.storage) + response = GitalyClient.call( + @repository.storage, :operation_service, :user_commit_files, req_enum, + timeout: GitalyClient.long_timeout, remote_storage: start_repository&.storage) if (pre_receive_error = response.pre_receive_error.presence) raise Gitlab::Git::PreReceiveError, pre_receive_error diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index 42f9c165610..bb6bc3121bd 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -7,7 +7,8 @@ module Gitlab TAGS_SORT_KEY = { 'name' => Gitaly::FindAllTagsRequest::SortBy::Key::REFNAME, - 'updated' => Gitaly::FindAllTagsRequest::SortBy::Key::CREATORDATE + 'updated' => Gitaly::FindAllTagsRequest::SortBy::Key::CREATORDATE, + 'version' => Gitaly::FindAllTagsRequest::SortBy::Key::VERSION_REFNAME }.freeze TAGS_SORT_DIRECTION = { @@ -104,7 +105,7 @@ module Gitlab return unless branch target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) - Gitlab::Git::Branch.new(@repository, encode!(branch.name.dup), branch.target_commit.id, target_commit) + Gitlab::Git::Branch.new(@repository, branch.name.dup, branch.target_commit.id, target_commit) end def find_tag(tag_name) @@ -258,7 +259,7 @@ module Gitlab end def sort_tags_by_param(sort_by) - match = sort_by.match(/^(?<key>name|updated)_(?<direction>asc|desc)$/) + match = sort_by.match(/^(?<key>name|updated|version)_(?<direction>asc|desc)$/) return unless match @@ -269,14 +270,23 @@ module Gitlab end def consume_find_local_branches_response(response) - response.flat_map do |message| - message.branches.map do |gitaly_branch| - Gitlab::Git::Branch.new( - @repository, - encode!(gitaly_branch.name.dup), - gitaly_branch.commit_id, - commit_from_local_branches_response(gitaly_branch) - ) + if Feature.enabled?(:gitaly_simplify_find_local_branches_response, type: :undefined) + response.flat_map do |message| + message.local_branches.map do |branch| + target_commit = Gitlab::Git::Commit.decorate(@repository, branch.target_commit) + Gitlab::Git::Branch.new(@repository, branch.name, branch.target_commit.id, target_commit) + end + end + else + response.flat_map do |message| + message.branches.map do |gitaly_branch| + Gitlab::Git::Branch.new( + @repository, + gitaly_branch.name.dup, + gitaly_branch.commit_id, + commit_from_local_branches_response(gitaly_branch) + ) + end end end end diff --git a/lib/gitlab/gitaly_client/server_service.rb b/lib/gitlab/gitaly_client/server_service.rb index 36bda67c26e..48fd0e66354 100644 --- a/lib/gitlab/gitaly_client/server_service.rb +++ b/lib/gitlab/gitaly_client/server_service.rb @@ -26,6 +26,19 @@ module Gitlab storage_specific(disk_statistics) end + def readiness_check + request = Gitaly::ReadinessCheckRequest.new(timeout: GitalyClient.medium_timeout) + response = GitalyClient.call(@storage, :server_service, :readiness_check, request, timeout: GitalyClient.default_timeout) + + return { success: true } if response.ok_response + + failed_checks = response.failure_response.failed_checks.map do |failed_check| + ["#{failed_check.name}: #{failed_check.error_message}"] + end + + { success: false, message: failed_checks.join("\n") } + end + private def storage_specific(response) diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb new file mode 100644 index 00000000000..b71d5f753f2 --- /dev/null +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class AttachmentsDownloader + include ::Gitlab::ImportExport::CommandLineUtil + include ::BulkImports::FileDownloads::FilenameFetch + include ::BulkImports::FileDownloads::Validations + + DownloadError = Class.new(StandardError) + + FILENAME_SIZE_LIMIT = 255 # chars before the extension + DEFAULT_FILE_SIZE_LIMIT = 25.megabytes + TMP_DIR = File.join(Dir.tmpdir, 'github_attachments').freeze + + attr_reader :file_url, :filename, :file_size_limit + + def initialize(file_url, file_size_limit: DEFAULT_FILE_SIZE_LIMIT) + @file_url = file_url + @file_size_limit = file_size_limit + + filename = URI(file_url).path.split('/').last + @filename = ensure_filename_size(filename) + end + + def perform + validate_content_length + validate_filepath + + file = download + validate_symlink + file + end + + def delete + FileUtils.rm_rf File.dirname(filepath) + end + + private + + def raise_error(message) + raise DownloadError, message + end + + def response_headers + @response_headers ||= + Gitlab::HTTP.perform_request(Net::HTTP::Head, file_url, {}).headers + end + + def download + file = File.open(filepath, 'wb') + Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, stream_body: true) { |batch| file.write(batch) } + file + end + + def filepath + @filepath ||= begin + dir = File.join(TMP_DIR, SecureRandom.uuid) + mkdir_p dir + File.join(dir, filename) + end + end + end + end +end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 11a41149274..6cff15a204f 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -76,11 +76,15 @@ module Gitlab each_object(:pull_request_reviews, repo_name, iid) end + def repos(options = {}) + octokit.repos(nil, options).map(&:to_h) + end + # Returns the details of a GitHub repository. # # name - The path (in the form `owner/repository`) of the repository. def repository(name) - with_rate_limit { octokit.repo(name) } + with_rate_limit { octokit.repo(name).to_h } end def pull_request(repo_name, iid) @@ -99,6 +103,14 @@ module Gitlab each_object(:releases, *args) end + def branches(*args) + each_object(:branches, *args) + end + + def branch_protection(repo_name, branch_name) + with_rate_limit { octokit.branch_protection(repo_name, branch_name) } + end + # Fetches data from the GitHub API and yields a Page object for every page # of data, without loading all of them into memory. # @@ -167,7 +179,7 @@ module Gitlab end def search_repos_by_name(name, options = {}) - with_retry { octokit.search_repositories(search_query(str: name, type: :name), options) } + with_retry { octokit.search_repositories(search_query(str: name, type: :name), options).to_h } end def search_query(str:, type:, include_collaborations: true, include_orgs: true) diff --git a/lib/gitlab/github_import/importer/events/base_importer.rb b/lib/gitlab/github_import/importer/events/base_importer.rb index 9ab1d916d33..8218acf2bfb 100644 --- a/lib/gitlab/github_import/importer/events/base_importer.rb +++ b/lib/gitlab/github_import/importer/events/base_importer.rb @@ -29,6 +29,19 @@ module Gitlab def issuable_db_id(object) IssuableFinder.new(project, object).database_id end + + def issuable_type(issue_event) + merge_request_event?(issue_event) ? MergeRequest.name : Issue.name + end + + def merge_request_event?(issue_event) + issue_event.issuable_type == MergeRequest.name + end + + def resource_event_belongs_to(issue_event) + belongs_to_key = merge_request_event?(issue_event) ? :merge_request_id : :issue_id + { belongs_to_key => issuable_db_id(issue_event) } + end end end end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index c8f6335e4a8..b75d41f40de 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -7,22 +7,22 @@ module Gitlab class ChangedAssignee < BaseImporter def execute(issue_event) assignee_id = author_id(issue_event, author_key: :assignee) - assigner_id = author_id(issue_event, author_key: :assigner) + author_id = author_id(issue_event, author_key: :actor) - note_body = parse_body(issue_event, assigner_id, assignee_id) + note_body = parse_body(issue_event, assignee_id) - create_note(issue_event, note_body, assigner_id) + create_note(issue_event, note_body, author_id) end private - def create_note(issue_event, note_body, assigner_id) + def create_note(issue_event, note_body, author_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, - author_id: assigner_id, + author_id: author_id, note: note_body, system_note_metadata: SystemNoteMetadata.new( { @@ -36,12 +36,14 @@ module Gitlab ) end - def parse_body(issue_event, assigner_id, assignee_id) + def parse_body(issue_event, assignee_id) + assignee = User.find(assignee_id).to_reference + Gitlab::I18n.with_default_locale do if issue_event.event == "unassigned" - "unassigned #{User.find(assigner_id).to_reference}" + "unassigned #{assignee}" else - "assigned to #{User.find(assignee_id).to_reference}" + "assigned to #{assignee}" end end end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 818a9202745..83130d18db9 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -12,13 +12,14 @@ module Gitlab private def create_event(issue_event) - ResourceLabelEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceLabelEvent.create!(attrs) end def label_finder diff --git a/lib/gitlab/github_import/importer/events/changed_milestone.rb b/lib/gitlab/github_import/importer/events/changed_milestone.rb index 3164c041dc3..39b92d88b58 100644 --- a/lib/gitlab/github_import/importer/events/changed_milestone.rb +++ b/lib/gitlab/github_import/importer/events/changed_milestone.rb @@ -17,14 +17,15 @@ module Gitlab private def create_event(issue_event) - ResourceMilestoneEvent.create!( - issue_id: issuable_db_id(issue_event), + attrs = { user_id: author_id(issue_event), created_at: issue_event.created_at, milestone_id: project.milestones.find_by_title(issue_event.milestone_title)&.id, action: action(issue_event.event), state: DEFAULT_STATE - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceMilestoneEvent.create!(attrs) end def action(event_type) diff --git a/lib/gitlab/github_import/importer/events/changed_reviewer.rb b/lib/gitlab/github_import/importer/events/changed_reviewer.rb new file mode 100644 index 00000000000..17b1fa4ab45 --- /dev/null +++ b/lib/gitlab/github_import/importer/events/changed_reviewer.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module Events + class ChangedReviewer < BaseImporter + def execute(issue_event) + requested_reviewer_id = author_id(issue_event, author_key: :requested_reviewer) + review_requester_id = author_id(issue_event, author_key: :review_requester) + + note_body = parse_body(issue_event, requested_reviewer_id) + + create_note(issue_event, note_body, review_requester_id) + end + + private + + def create_note(issue_event, note_body, review_requester_id) + Note.create!( + system: true, + noteable_type: issuable_type(issue_event), + noteable_id: issuable_db_id(issue_event), + project: project, + author_id: review_requester_id, + note: note_body, + system_note_metadata: SystemNoteMetadata.new( + { + action: 'reviewer', + created_at: issue_event.created_at, + updated_at: issue_event.created_at + } + ), + created_at: issue_event.created_at, + updated_at: issue_event.created_at + ) + end + + def parse_body(issue_event, requested_reviewer_id) + requested_reviewer = User.find(requested_reviewer_id).to_reference + + if issue_event.event == 'review_request_removed' + "#{SystemNotes::IssuablesService.issuable_events[:review_request_removed]}" \ + " #{requested_reviewer}" + else + "#{SystemNotes::IssuablesService.issuable_events[:review_requested]}" \ + " #{requested_reviewer}" + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/events/closed.rb b/lib/gitlab/github_import/importer/events/closed.rb index ca8730d0f27..58d9dbf826c 100644 --- a/lib/gitlab/github_import/importer/events/closed.rb +++ b/lib/gitlab/github_import/importer/events/closed.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'closed', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,15 +25,16 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), source_commit: issue_event.commit_id, state: 'closed', close_after_error_tracking_resolve: false, close_auto_resolve_prometheus_alert: false, created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/events/cross_referenced.rb b/lib/gitlab/github_import/importer/events/cross_referenced.rb index 89fc1bdeb09..b56ae186d3c 100644 --- a/lib/gitlab/github_import/importer/events/cross_referenced.rb +++ b/lib/gitlab/github_import/importer/events/cross_referenced.rb @@ -33,7 +33,7 @@ module Gitlab def create_note(issue_event, note_body, user_id) Note.create!( system: true, - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), noteable_id: issuable_db_id(issue_event), project: project, author_id: user_id, diff --git a/lib/gitlab/github_import/importer/events/renamed.rb b/lib/gitlab/github_import/importer/events/renamed.rb index 96d112b04c6..fb9e08116ba 100644 --- a/lib/gitlab/github_import/importer/events/renamed.rb +++ b/lib/gitlab/github_import/importer/events/renamed.rb @@ -14,7 +14,7 @@ module Gitlab def note_params(issue_event) { noteable_id: issuable_db_id(issue_event), - noteable_type: Issue.name, + noteable_type: issuable_type(issue_event), project_id: project.id, author_id: author_id(issue_event), note: parse_body(issue_event), diff --git a/lib/gitlab/github_import/importer/events/reopened.rb b/lib/gitlab/github_import/importer/events/reopened.rb index b75344bf817..8abeba0777d 100644 --- a/lib/gitlab/github_import/importer/events/reopened.rb +++ b/lib/gitlab/github_import/importer/events/reopened.rb @@ -17,7 +17,7 @@ module Gitlab project_id: project.id, author_id: author_id(issue_event), action: 'reopened', - target_type: Issue.name, + target_type: issuable_type(issue_event), target_id: issuable_db_id(issue_event), created_at: issue_event.created_at, updated_at: issue_event.created_at @@ -25,12 +25,13 @@ module Gitlab end def create_state_event(issue_event) - ResourceStateEvent.create!( + attrs = { user_id: author_id(issue_event), - issue_id: issuable_db_id(issue_event), state: 'reopened', created_at: issue_event.created_at - ) + }.merge(resource_event_belongs_to(issue_event)) + + ResourceStateEvent.create!(attrs) end end end diff --git a/lib/gitlab/github_import/importer/issue_event_importer.rb b/lib/gitlab/github_import/importer/issue_event_importer.rb index ef456e56ee1..80749aae93c 100644 --- a/lib/gitlab/github_import/importer/issue_event_importer.rb +++ b/lib/gitlab/github_import/importer/issue_event_importer.rb @@ -15,11 +15,7 @@ module Gitlab @client = client end - # TODO: Add MergeRequest events support - # https://gitlab.com/groups/gitlab-org/-/epics/7673 def execute - return if issue_event.issuable_type == 'MergeRequest' - importer = event_importer_class(issue_event) if importer importer.new(project, client).execute(issue_event) @@ -49,6 +45,8 @@ module Gitlab Gitlab::GithubImport::Importer::Events::CrossReferenced when 'assigned', 'unassigned' Gitlab::GithubImport::Importer::Events::ChangedAssignee + when 'review_requested', 'review_request_removed' + Gitlab::GithubImport::Importer::Events::ChangedReviewer end end end diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb new file mode 100644 index 00000000000..16215fdce8e --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchImporter + attr_reader :protected_branch, :project, :client + + # protected_branch - An instance of + # `Gitlab::GithubImport::Representation::ProtectedBranch`. + # project - An instance of `Project` + # client - An instance of `Gitlab::GithubImport::Client` + def initialize(protected_branch, project, client) + @protected_branch = protected_branch + @project = project + @client = client + end + + def execute + # The creator of the project is always allowed to create protected + # branches, so we skip the authorization check in this service class. + ProtectedBranches::CreateService + .new(project, project.creator, params) + .execute(skip_authorization: true) + end + + private + + def params + { + name: protected_branch.id, + push_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + merge_access_levels_attributes: [{ access_level: Gitlab::Access::MAINTAINER }], + allow_force_push: allow_force_push? + } + end + + def allow_force_push? + if ProtectedBranch.protected?(project, protected_branch.id) + ProtectedBranch.allow_force_push?(project, protected_branch.id) && protected_branch.allow_force_pushes + else + protected_branch.allow_force_pushes + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb new file mode 100644 index 00000000000..b5be823d5ab --- /dev/null +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ProtectedBranchesImporter + include ParallelScheduling + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + repo = project.import_source + + protected_branches = client.branches(repo).select { |branch| branch.protection&.enabled } + protected_branches.each do |protected_branch| + object = client.branch_protection(repo, protected_branch.name) + next if object.nil? || already_imported?(object) + + yield object + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + mark_as_imported(object) + end + end + + def importer_class + ProtectedBranchImporter + end + + def representation_class + Gitlab::GithubImport::Representation::ProtectedBranch + end + + def sidekiq_worker_class + ImportProtectedBranchWorker + end + + def object_type + :protected_branch + end + + def collection_method + :protected_branches + end + + def id_for_already_imported_cache(protected_branch) + protected_branch.name + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/release_attachments_importer.rb b/lib/gitlab/github_import/importer/release_attachments_importer.rb new file mode 100644 index 00000000000..6419851623c --- /dev/null +++ b/lib/gitlab/github_import/importer/release_attachments_importer.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleaseAttachmentsImporter + attr_reader :release_db_id, :release_description, :project + + # release - An instance of `ReleaseAttachments`. + # project - An instance of `Project`. + # client - An instance of `Gitlab::GithubImport::Client`. + def initialize(release_attachments, project, _client = nil) + @release_db_id = release_attachments.release_db_id + @release_description = release_attachments.description + @project = project + end + + def execute + attachment_urls = MarkdownText.fetch_attachment_urls(release_description) + new_description = attachment_urls.reduce(release_description) do |description, url| + new_url = download_attachment(url) + description.gsub(url, new_url) + end + + Release.find(release_db_id).update_column(:description, new_description) + end + + private + + # in: github attachment markdown url + # out: gitlab attachment markdown url + def download_attachment(markdown_url) + url = extract_url_from_markdown(markdown_url) + name_prefix = extract_name_from_markdown(markdown_url) + + downloader = ::Gitlab::GithubImport::AttachmentsDownloader.new(url) + file = downloader.perform + uploader = UploadService.new(project, file, FileUploader).execute + "#{name_prefix}(#{uploader.to_h[:url]})" + ensure + downloader&.delete + end + + # in: "" + # out: https://user-images.githubusercontent.com/.. + def extract_url_from_markdown(text) + text.match(%r{https://.*\)$}).to_a[0].chop + end + + # in: "" + # out: ![image-icon] + def extract_name_from_markdown(text) + text.match(%r{^!?\[.*\]}).to_a[0] + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/releases_attachments_importer.rb b/lib/gitlab/github_import/importer/releases_attachments_importer.rb new file mode 100644 index 00000000000..7221c802d83 --- /dev/null +++ b/lib/gitlab/github_import/importer/releases_attachments_importer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + class ReleasesAttachmentsImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + # The method that will be called for traversing through all the objects to + # import, yielding them to the supplied block. + def each_object_to_import + project.releases.select(:id, :description).each_batch(of: BATCH_SIZE, column: :id) do |batch| + batch.each do |release| + next if already_imported?(release) + + Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) + + yield release + + # We mark the object as imported immediately so we don't end up + # scheduling it multiple times. + mark_as_imported(release) + end + end + end + + def representation_class + Representation::ReleaseAttachments + end + + def importer_class + ReleaseAttachmentsImporter + end + + def sidekiq_worker_class + ImportReleaseAttachmentsWorker + end + + def collection_method + :release_attachments + end + + def object_type + :release_attachment + end + + def id_for_already_imported_cache(release) + release.id + end + + def object_representation(object) + representation_class.from_db_record(object) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index aba4729e9c8..708768a60cf 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -17,7 +17,7 @@ module Gitlab # Returns true if we should import the wiki for the project. # rubocop: disable CodeReuse/ActiveRecord def import_wiki? - client_repository&.has_wiki && + client_repository[:has_wiki] && !project.wiki_repository_exists? && Gitlab::GitalyClient::RemoteService.exists?(wiki_url) end @@ -86,7 +86,7 @@ module Gitlab private def default_branch - client_repository&.default_branch + client_repository[:default_branch] end def client_repository diff --git a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb index 8e4015acbbc..8a9ddfc6ec0 100644 --- a/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb +++ b/lib/gitlab/github_import/importer/single_endpoint_issue_events_importer.rb @@ -7,7 +7,7 @@ module Gitlab include ParallelScheduling include SingleEndpointNotesImporting - PROCESSED_PAGE_CACHE_KEY = 'issues/%{issue_iid}/%{collection}' + PROCESSED_PAGE_CACHE_KEY = 'issues/%{issuable_iid}/%{collection}' BATCH_SIZE = 100 def initialize(project, client, parallel: true) @@ -27,12 +27,20 @@ module Gitlab Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - associated.issue = { 'number' => parent_record.iid } + pull_request = parent_record.is_a? MergeRequest + associated.issue = { 'number' => parent_record.iid, 'pull_request' => pull_request } yield(associated) mark_as_imported(associated) end + # In Github Issues and MergeRequests uses the same API to get their events. + # Even more - they have commonly uniq iid + def each_associated_page(&block) + issues_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + merge_requests_collection.each_batch(of: BATCH_SIZE, column: :iid) { |batch| process_batch(batch, &block) } + end + def importer_class IssueEventImporter end @@ -53,16 +61,20 @@ module Gitlab :issue_timeline end - def parent_collection + def issues_collection project.issues.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord end + def merge_requests_collection + project.merge_requests.where.not(iid: already_imported_parents).select(:id, :iid) # rubocop: disable CodeReuse/ActiveRecord + end + def parent_imported_cache_key "github-importer/issues/#{collection_method}/already-imported/#{project.id}" end - def page_counter_id(issue) - PROCESSED_PAGE_CACHE_KEY % { issue_iid: issue.iid, collection: collection_method } + def page_counter_id(issuable) + PROCESSED_PAGE_CACHE_KEY % { issuable_iid: issuable.iid, collection: collection_method } end def id_for_already_imported_cache(event) @@ -74,10 +86,10 @@ module Gitlab end # Cross-referenced events on Github doesn't have id. - def compose_associated_id!(issue, event) + def compose_associated_id!(issuable, event) return if event.event != 'cross-referenced' - event.id = "cross-reference##{issue.id}-in-#{event.source.issue.id}" + event.id = "cross-reference##{issuable.iid}-in-#{event.source.issue.id}" end end end diff --git a/lib/gitlab/github_import/markdown_text.rb b/lib/gitlab/github_import/markdown_text.rb index 692016bd005..bf2856bc77f 100644 --- a/lib/gitlab/github_import/markdown_text.rb +++ b/lib/gitlab/github_import/markdown_text.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# This class includes overriding Kernel#format method +# what makes impossible to use it here +# rubocop:disable Style/FormatString module Gitlab module GithubImport class MarkdownText @@ -8,6 +11,21 @@ module Gitlab ISSUE_REF_MATCHER = '%{github_url}/%{import_source}/issues' PULL_REF_MATCHER = '%{github_url}/%{import_source}/pull' + MEDIA_TYPES = %w[gif jpeg jpg mov mp4 png svg webm].freeze + DOC_TYPES = %w[ + csv docx fodg fodp fods fodt gz log md odf odg odp ods + odt pdf pptx tgz txt xls xlsx zip + ].freeze + ALL_TYPES = (MEDIA_TYPES + DOC_TYPES).freeze + + # On github.com we have base url for docs and CDN url for media. + # On github EE as far as we can know there is no CDN urls and media is placed on base url. + # To no escape the escaping symbol we use single quotes instead of double with interpolation. + # rubocop:disable Style/StringConcatenation + CDN_URL_MATCHER = '(!\[.+\]\(%{github_media_cdn}/\d+/(\w|-)+\.(' + MEDIA_TYPES.join('|') + ')\))' + BASE_URL_MATCHER = '(\[.+\]\(%{github_url}/.+/.+/files/\d+/.+\.(' + ALL_TYPES.join('|') + ')\))' + # rubocop:enable Style/StringConcatenation + class << self def format(*args) new(*args).to_s @@ -24,8 +42,20 @@ module Gitlab .gsub(pull_ref_matcher, url_helpers.project_merge_requests_url(project)) end + def fetch_attachment_urls(text) + cdn_url_matcher = CDN_URL_MATCHER % { github_media_cdn: Regexp.escape(github_media_cdn) } + doc_url_matcher = BASE_URL_MATCHER % { github_url: Regexp.escape(github_url) } + + text.scan(Regexp.new(cdn_url_matcher)).map(&:first) + + text.scan(Regexp.new(doc_url_matcher)).map(&:first) + end + private + def github_media_cdn + 'https://user-images.githubusercontent.com' + end + # Returns github domain without slash in the end def github_url oauth_config = Gitlab::Auth::OAuth::Provider.config_for('github') || {} @@ -63,3 +93,4 @@ module Gitlab end end end +# rubocop:enable Style/FormatString diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index a8c18c74d24..bf5046de36c 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -63,7 +63,7 @@ module Gitlab # Imports all the objects in sequence in the current thread. def sequential_import each_object_to_import do |object| - repr = representation_class.from_api_response(object, additional_object_data) + repr = object_representation(object) importer_class.new(repr, project, client).execute end @@ -83,7 +83,7 @@ module Gitlab import_arguments = [] each_object_to_import do |object| - repr = representation_class.from_api_response(object, additional_object_data) + repr = object_representation(object) import_arguments << [project.id, repr.to_hash, waiter.key] @@ -210,6 +210,10 @@ module Gitlab {} end + def object_representation(object) + representation_class.from_api_response(object, additional_object_data) + end + def info(project_id, extra = {}) Logger.info(log_attributes(project_id, extra)) end diff --git a/lib/gitlab/github_import/representation/expose_attribute.rb b/lib/gitlab/github_import/representation/expose_attribute.rb index d2438ee8094..84de4d4798d 100644 --- a/lib/gitlab/github_import/representation/expose_attribute.rb +++ b/lib/gitlab/github_import/representation/expose_attribute.rb @@ -20,6 +20,10 @@ module Gitlab end end end + + def [](key) + respond_to?(key.to_sym) ? attributes[key] : nil + end end end end diff --git a/lib/gitlab/github_import/representation/issue_event.rb b/lib/gitlab/github_import/representation/issue_event.rb index 67a5df73a97..89271a7dcd6 100644 --- a/lib/gitlab/github_import/representation/issue_event.rb +++ b/lib/gitlab/github_import/representation/issue_event.rb @@ -10,7 +10,8 @@ module Gitlab attr_reader :attributes expose_attribute :id, :actor, :event, :commit_id, :label_title, :old_title, :new_title, - :milestone_title, :issue, :source, :assignee, :assigner, :created_at + :milestone_title, :issue, :source, :assignee, :review_requester, + :requested_reviewer, :created_at # attributes - A Hash containing the event details. The keys of this # Hash (and any nested hashes) must be symbols. @@ -47,7 +48,8 @@ module Gitlab issue: event.issue&.to_h&.symbolize_keys, source: event.source, assignee: user_representation(event.assignee), - assigner: user_representation(event.assigner), + requested_reviewer: user_representation(event.requested_reviewer), + review_requester: user_representation(event.review_requester), created_at: event.created_at ) end @@ -57,7 +59,8 @@ module Gitlab hash = Representation.symbolize_hash(raw_hash) hash[:actor] = user_representation(hash[:actor], source: :hash) hash[:assignee] = user_representation(hash[:assignee], source: :hash) - hash[:assigner] = user_representation(hash[:assigner], source: :hash) + hash[:requested_reviewer] = user_representation(hash[:requested_reviewer], source: :hash) + hash[:review_requester] = user_representation(hash[:review_requester], source: :hash) new(hash) end diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb new file mode 100644 index 00000000000..b80b7cf1076 --- /dev/null +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + class ProtectedBranch + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :id, :allow_force_pushes + + # Builds a Branch Protection info from a GitHub API response. + # Resource structure details: + # https://docs.github.com/en/rest/branches/branch-protection#get-branch-protection + # branch_protection - An instance of `Sawyer::Resource` containing the protection details. + def self.from_api_response(branch_protection, _additional_object_data = {}) + branch_name = branch_protection.url.match(%r{/branches/(\S{1,255})/protection$})[1] + + hash = { + id: branch_name, + allow_force_pushes: branch_protection.allow_force_pushes.enabled + } + + new(hash) + end + + # Builds a new Protection using a Hash that was built from a JSON payload. + def self.from_json_hash(raw_hash) + new(Representation.symbolize_hash(raw_hash)) + end + + # attributes - A Hash containing the raw Protection details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { id: id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/representation/release_attachments.rb b/lib/gitlab/github_import/representation/release_attachments.rb new file mode 100644 index 00000000000..fd272be2405 --- /dev/null +++ b/lib/gitlab/github_import/representation/release_attachments.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +# This class only partly represents Release record from DB and +# is used to connect ReleasesAttachmentsImporter with ReleaseAttachmentsImporter +# without modifying ObjectImporter a lot. +# Attachments are inside release's `description`. +module Gitlab + module GithubImport + module Representation + class ReleaseAttachments + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :release_db_id, :description + + # Builds a event from a GitHub API response. + # + # release - An instance of `Release` model. + def self.from_db_record(release) + new( + release_db_id: release.id, + description: release.description + ) + end + + def self.from_json_hash(raw_hash) + new Representation.symbolize_hash(raw_hash) + end + + # attributes - A Hash containing the event details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { db_id: release_db_id } + end + end + end + end +end diff --git a/lib/gitlab/github_import/sequential_importer.rb b/lib/gitlab/github_import/sequential_importer.rb index 6bc37337799..ab37bc92ee7 100644 --- a/lib/gitlab/github_import/sequential_importer.rb +++ b/lib/gitlab/github_import/sequential_importer.rb @@ -16,6 +16,7 @@ module Gitlab ].freeze PARALLEL_IMPORTERS = [ + Importer::ProtectedBranchesImporter, Importer::PullRequestsImporter, Importer::IssuesImporter, Importer::DiffNotesImporter, diff --git a/lib/gitlab/github_import/single_endpoint_notes_importing.rb b/lib/gitlab/github_import/single_endpoint_notes_importing.rb index 0a3559adde3..aea4059dfbc 100644 --- a/lib/gitlab/github_import/single_endpoint_notes_importing.rb +++ b/lib/gitlab/github_import/single_endpoint_notes_importing.rb @@ -63,23 +63,27 @@ module Gitlab mark_as_imported(associated) end - def each_associated_page + def each_associated_page(&block) parent_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| - batch.each do |parent_record| - # The page counter needs to be scoped by parent_record to avoid skipping - # pages of notes from already imported parent_record. - page_counter = PageCounter.new(project, page_counter_id(parent_record)) - repo = project.import_source - options = collection_options.merge(page: page_counter.current) + process_batch(batch, &block) + end + end - client.each_page(collection_method, repo, parent_record.iid, options) do |page| - next unless page_counter.set(page.number) + def process_batch(batch) + batch.each do |parent_record| + # The page counter needs to be scoped by parent_record to avoid skipping + # pages of notes from already imported parent_record. + page_counter = PageCounter.new(project, page_counter_id(parent_record)) + repo = project.import_source + options = collection_options.merge(page: page_counter.current) - yield parent_record, page - end + client.each_page(collection_method, repo, parent_record.iid, options) do |page| + next unless page_counter.set(page.number) - mark_parent_imported(parent_record) + yield parent_record, page end + + mark_parent_imported(parent_record) end end diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index 6d6a00d260d..1feb0d450f0 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -45,8 +45,10 @@ module Gitlab object&.actor when :assignee object&.assignee - when :assigner - object&.assigner + when :requested_reviewer + object&.requested_reviewer + when :review_requester + object&.review_requester else object&.author end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 5f1802e323c..08a614edb4b 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -56,7 +56,6 @@ module Gitlab push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:gl_avatar_for_all_user_avatars) - push_frontend_feature_flag(:mr_attention_requests, current_user) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/graphql/errors.rb b/lib/gitlab/graphql/errors.rb index 40b90310e8b..657364abfdf 100644 --- a/lib/gitlab/graphql/errors.rb +++ b/lib/gitlab/graphql/errors.rb @@ -7,6 +7,7 @@ module Gitlab ArgumentError = Class.new(BaseError) ResourceNotAvailable = Class.new(BaseError) MutationError = Class.new(BaseError) + LimitError = Class.new(BaseError) end end end diff --git a/lib/gitlab/graphql/limit/field_call_count.rb b/lib/gitlab/graphql/limit/field_call_count.rb new file mode 100644 index 00000000000..4165970a2a6 --- /dev/null +++ b/lib/gitlab/graphql/limit/field_call_count.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Gitlab + module Graphql + module Limit + class FieldCallCount < ::GraphQL::Schema::FieldExtension + def resolve(object:, arguments:, context:) + raise Gitlab::Graphql::Errors::ArgumentError, 'Limit must be specified.' unless limit + raise Gitlab::Graphql::Errors::LimitError, error_message if increment_call_count(context) > limit + + yield(object, arguments) + end + + private + + def increment_call_count(context) + context[:call_count] ||= {} + context[:call_count][field] ||= 0 + context[:call_count][field] += 1 + end + + def limit + options[:limit] + end + + def error_message + "\"#{field.graphql_name}\" field can be requested only for #{limit} #{field.owner.graphql_name}(s) at a time." + end + end + end + end +end diff --git a/lib/gitlab/graphql/pagination/keyset/connection.rb b/lib/gitlab/graphql/pagination/keyset/connection.rb index b074c273996..987a5e7b74b 100644 --- a/lib/gitlab/graphql/pagination/keyset/connection.rb +++ b/lib/gitlab/graphql/pagination/keyset/connection.rb @@ -59,11 +59,15 @@ module Gitlab if before true elsif first - case sliced_nodes - when Array - sliced_nodes.size > limit_value + if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + limited_nodes.size > limit_value else - sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord + case sliced_nodes + when Array + sliced_nodes.size > limit_value + else + sliced_nodes.limit(1).offset(limit_value).exists? # rubocop: disable CodeReuse/ActiveRecord + end end else false @@ -89,7 +93,7 @@ module Gitlab # So we're ok loading them into memory here as that's bound to happen # anyway. Having them ready means we can modify the result while # rendering the fields. - @nodes ||= limited_nodes.to_a + @nodes ||= limited_nodes.to_a.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord end def items @@ -116,15 +120,21 @@ module Gitlab end if last - paginated_nodes = LastItems.take_items(sliced_nodes, limit_value + 1) + paginated_nodes = sliced_nodes.last(limit_value + 1) # there is an extra node, so there is a previous page @has_previous_page = paginated_nodes.count > limit_value @has_previous_page ? paginated_nodes.last(limit_value) : paginated_nodes elsif loaded?(sliced_nodes) - sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord + if Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + sliced_nodes.take(limit_value + 1) # rubocop: disable CodeReuse/ActiveRecord + else + sliced_nodes.take(limit_value) # rubocop: disable CodeReuse/ActiveRecord + end + elsif Feature.enabled?(:graphql_keyset_pagination_without_next_page_query) + sliced_nodes.limit(limit_value + 1).to_a else - sliced_nodes.limit(limit_value) # rubocop: disable CodeReuse/ActiveRecord + sliced_nodes.limit(limit_value) end end end @@ -141,7 +151,7 @@ module Gitlab def limit_value # note: only first _or_ last can be specified, not both - @limit_value ||= [first, last, max_page_size].compact.min + @limit_value ||= [first, last, max_page_size, GitlabSchema.default_max_page_size].compact.min end def loaded?(items) diff --git a/lib/gitlab/graphql/pagination/keyset/last_items.rb b/lib/gitlab/graphql/pagination/keyset/last_items.rb deleted file mode 100644 index 960567a6fbc..00000000000 --- a/lib/gitlab/graphql/pagination/keyset/last_items.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Graphql - module Pagination - module Keyset - # This class handles the last(N) ActiveRecord call even if a special ORDER BY configuration is present. - # For the last(N) call, ActiveRecord calls reverse_order, however for some cases it raises - # ActiveRecord::IrreversibleOrderError error. - class LastItems - # rubocop: disable CodeReuse/ActiveRecord - def self.take_items(scope, count) - if Gitlab::Pagination::Keyset::Order.keyset_aware?(scope) - order = Gitlab::Pagination::Keyset::Order.extract_keyset_order_object(scope) - items = scope.reorder(order.reversed_order).first(count) - items.is_a?(Array) ? items.reverse : items - else - scope.last(count) - end - end - end - end - end - end -end diff --git a/lib/gitlab/graphql/type_name_deprecations.rb b/lib/gitlab/graphql/type_name_deprecations.rb index c27ad1d54f5..1ec6fd1c09f 100644 --- a/lib/gitlab/graphql/type_name_deprecations.rb +++ b/lib/gitlab/graphql/type_name_deprecations.rb @@ -14,6 +14,9 @@ module Gitlab DEPRECATIONS = [ Gitlab::Graphql::DeprecationsBase::NameDeprecation.new( old_name: 'CiRunnerUpgradeStatusType', new_name: 'CiRunnerUpgradeStatus', milestone: '15.3' + ), + Gitlab::Graphql::DeprecationsBase::NameDeprecation.new( + old_name: 'RunnerMembershipFilter', new_name: 'CiRunnerMembershipFilter', milestone: '15.4' ) ].freeze diff --git a/lib/gitlab/harbor/query.rb b/lib/gitlab/harbor/query.rb index c120810ecf1..fcd984b01ce 100644 --- a/lib/gitlab/harbor/query.rb +++ b/lib/gitlab/harbor/query.rb @@ -17,7 +17,7 @@ module Gitlab message: 'Id invalid' }, allow_blank: true validates :artifact_id, format: { - with: /\A[a-zA-Z0-9\_\.\-$]+\z/, + with: /\A[a-zA-Z0-9\_\.\-$:]+\z/, message: 'Id invalid' }, allow_blank: true validates :sort, format: { diff --git a/lib/gitlab/health_checks/gitaly_check.rb b/lib/gitlab/health_checks/gitaly_check.rb index f5f142c251f..2bd8ea711b5 100644 --- a/lib/gitlab/health_checks/gitaly_check.rb +++ b/lib/gitlab/health_checks/gitaly_check.rb @@ -27,17 +27,35 @@ module Gitlab end def check(storage_name) - serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) - result = serv.check + storage_healthy = healthy(storage_name) + unless storage_healthy[:success] + return HealthChecks::Result.new( + name, + storage_healthy[:success], + storage_healthy[:message], + shard: storage_name + ) + end + storage_ready = ready(storage_name) HealthChecks::Result.new( name, - result[:success], - result[:message], + storage_ready[:success], + storage_ready[:message], shard: storage_name ) end + def healthy(storage_name) + serv = Gitlab::GitalyClient::HealthCheckService.new(storage_name) + serv.check + end + + def ready(storage_name) + serv = Gitlab::GitalyClient::ServerService.new(storage_name) + serv.readiness_check + end + private def metric_prefix diff --git a/lib/gitlab/health_checks/redis.rb b/lib/gitlab/health_checks/redis.rb new file mode 100644 index 00000000000..895bce5a5a9 --- /dev/null +++ b/lib/gitlab/health_checks/redis.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module HealthChecks + module Redis + ALL_INSTANCE_CHECKS = + ::Gitlab::Redis::ALL_CLASSES.map do |instance_class| + check_class = Class.new + check_class.extend(RedisAbstractCheck) + const_set("#{instance_class.store_name}Check", check_class) + + check_class + end + end + end +end diff --git a/lib/gitlab/health_checks/redis/cache_check.rb b/lib/gitlab/health_checks/redis/cache_check.rb deleted file mode 100644 index bd843bdaac4..00000000000 --- a/lib/gitlab/health_checks/redis/cache_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class CacheCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/queues_check.rb b/lib/gitlab/health_checks/redis/queues_check.rb deleted file mode 100644 index fb92db937dc..00000000000 --- a/lib/gitlab/health_checks/redis/queues_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class QueuesCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/rate_limiting_check.rb b/lib/gitlab/health_checks/redis/rate_limiting_check.rb deleted file mode 100644 index 0e9d94f7dff..00000000000 --- a/lib/gitlab/health_checks/redis/rate_limiting_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class RateLimitingCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/redis_abstract_check.rb b/lib/gitlab/health_checks/redis/redis_abstract_check.rb index ecad4b06ea9..9a9a4d1faba 100644 --- a/lib/gitlab/health_checks/redis/redis_abstract_check.rb +++ b/lib/gitlab/health_checks/redis/redis_abstract_check.rb @@ -10,12 +10,12 @@ module Gitlab successful?(check) end - private - def redis_instance_class_name Gitlab::Redis.const_get(redis_instance_name.camelize, false) end + private + def metric_prefix "redis_#{redis_instance_name}_ping" end diff --git a/lib/gitlab/health_checks/redis/redis_check.rb b/lib/gitlab/health_checks/redis/redis_check.rb deleted file mode 100644 index c793a939abd..00000000000 --- a/lib/gitlab/health_checks/redis/redis_check.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class RedisCheck - extend SimpleAbstractCheck - - class << self - private - - def metric_prefix - 'redis_ping' - end - - def successful?(result) - result == true - end - - def check - redis_health_checks.all?(&:check_up) - end - - def redis_health_checks - [ - Gitlab::HealthChecks::Redis::CacheCheck, - Gitlab::HealthChecks::Redis::QueuesCheck, - Gitlab::HealthChecks::Redis::SharedStateCheck, - Gitlab::HealthChecks::Redis::TraceChunksCheck, - Gitlab::HealthChecks::Redis::RateLimitingCheck, - Gitlab::HealthChecks::Redis::SessionsCheck - ] - end - end - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/sessions_check.rb b/lib/gitlab/health_checks/redis/sessions_check.rb deleted file mode 100644 index 90a4c868f40..00000000000 --- a/lib/gitlab/health_checks/redis/sessions_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class SessionsCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/shared_state_check.rb b/lib/gitlab/health_checks/redis/shared_state_check.rb deleted file mode 100644 index 80f91784b8c..00000000000 --- a/lib/gitlab/health_checks/redis/shared_state_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class SharedStateCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/health_checks/redis/trace_chunks_check.rb b/lib/gitlab/health_checks/redis/trace_chunks_check.rb deleted file mode 100644 index 9a89a1ce51d..00000000000 --- a/lib/gitlab/health_checks/redis/trace_chunks_check.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module HealthChecks - module Redis - class TraceChunksCheck - extend RedisAbstractCheck - end - end - end -end diff --git a/lib/gitlab/hook_data/project_member_builder.rb b/lib/gitlab/hook_data/project_member_builder.rb index 90fc83fdf21..2ee61705ec1 100644 --- a/lib/gitlab/hook_data/project_member_builder.rb +++ b/lib/gitlab/hook_data/project_member_builder.rb @@ -37,16 +37,16 @@ module Gitlab project = project_member.project || Project.unscoped.find(project_member.source_id) { - project_name: project.name, - project_path: project.path, - project_path_with_namespace: project.full_path, - project_id: project.id, - user_username: project_member.user.username, - user_name: project_member.user.name, - user_email: project_member.user.email, - user_id: project_member.user.id, - access_level: project_member.human_access, - project_visibility: project.visibility + project_name: project.name, + project_path: project.path, + project_path_with_namespace: project.full_path, + project_id: project.id, + user_username: project_member.user.username, + user_name: project_member.user.name, + user_email: project_member.user.email, + user_id: project_member.user.id, + access_level: project_member.human_access, + project_visibility: project.visibility } end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 30465ff5f74..5b9216c0914 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 39, + 'da_DK' => 38, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 38, + 'es' => 37, 'fil_PH' => 0, 'fr' => 11, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 32, - 'ko' => 12, + 'ja' => 31, + 'ko' => 17, 'nb_NO' => 26, 'nl_NL' => 0, 'pl_PL' => 4, - 'pt_BR' => 55, - 'ro_RO' => 100, + 'pt_BR' => 56, + 'ro_RO' => 99, 'ru' => 27, 'si_LK' => 10, - 'tr_TR' => 12, + 'tr_TR' => 11, 'uk' => 50, - 'zh_CN' => 99, + 'zh_CN' => 97, 'zh_HK' => 1, - 'zh_TW' => 100 + 'zh_TW' => 99 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 4abc3da1190..8843b4f5755 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -12,6 +12,7 @@ module Gitlab @methods = config[:methods] || {} @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} + @include_if_exportable = config[:include_if_exportable] || {} end def find_root(model_key) @@ -35,7 +36,8 @@ module Gitlab methods: @methods[model_key], include: resolve_model_tree(model_tree), preload: resolve_preloads(model_key, model_tree), - export_reorder: @export_reorders[model_key] + export_reorder: @export_reorders[model_key], + include_if_exportable: @include_if_exportable[model_key] }.compact end diff --git a/lib/gitlab/import_export/base/relation_factory.rb b/lib/gitlab/import_export/base/relation_factory.rb index 1cbfcbdb595..bbec473d29d 100644 --- a/lib/gitlab/import_export/base/relation_factory.rb +++ b/lib/gitlab/import_export/base/relation_factory.rb @@ -31,6 +31,8 @@ module Gitlab TOKEN_RESET_MODELS = %i[Project Namespace Group Ci::Trigger Ci::Build Ci::Runner ProjectHook ErrorTracking::ProjectErrorTrackingSetting].freeze + attr_reader :relation_name, :importable + def self.create(*args, **kwargs) new(*args, **kwargs).create end diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index ea989487ebd..3c473449ec0 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -58,8 +58,19 @@ module Gitlab records.each_slice(BATCH_SIZE) do |batch| valid_records, invalid_records = batch.partition { |record| record.valid? } - invalid_subrelations << invalid_records relation_object.public_send(relation_name) << valid_records + + # Attempt to save some of the invalid subrelations, as they might be valid after all. + # For example, a merge request `Approval` validates presence of merge_request_id. + # It is not present at a time of calling `#valid?` above, since it's indeed missing. + # However, when saving such subrelation against already persisted merge request + # such validation won't fail (e.g. `merge_request.approvals << Approval.new(user_id: 1)`), + # as we're operating on a merge request that has `id` present. + invalid_records.each do |invalid_record| + relation_object.public_send(relation_name) << invalid_record + + invalid_subrelations << invalid_record unless invalid_record.persisted? + end end end end diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index 8df5d52bf77..a08efdf400b 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -27,6 +27,26 @@ included_attributes: - :name namespace_settings: - :prevent_sharing_groups_outside_hierarchy + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :title + - :description + - :sequence + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -44,6 +64,23 @@ excluded_attributes: - :max_pages_size epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + iterations: *iteration_excluded_definition methods: labels: @@ -83,6 +120,7 @@ ee: - events: - :push_event_payload - :system_note_metadata + - :resource_state_events - boards: - :board_assignee - :milestone @@ -92,3 +130,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_import_export.yml b/lib/gitlab/import_export/group/legacy_import_export.yml index 082d2346f3d..6507def7d01 100644 --- a/lib/gitlab/import_export/group/legacy_import_export.yml +++ b/lib/gitlab/import_export/group/legacy_import_export.yml @@ -24,6 +24,29 @@ included_attributes: - :username author: - :name + iterations_cadence: &iterations_cadence_definition + - :group_id + - :created_at + - :updated_at + - :start_date + - :next_run_date + - :duration_in_weeks + - :iterations_in_advance + - :active + - :automatic + - :roll_over + - :title + - :description + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_definition + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations: *iteration_definition excluded_attributes: group: @@ -41,6 +64,18 @@ excluded_attributes: - :extra_shared_runners_minutes_limit epics: - :state_id + iterations_cadence: &iterations_cadence_definition + - :id + iterations_cadences: *iterations_cadence_definition + iteration: &iteration_excluded_definition + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + iterations: *iteration_excluded_definition methods: labels: @@ -79,6 +114,7 @@ ee: - :award_emoji - events: - :push_event_payload + - :resource_state_events - boards: - :board_assignee - :milestone @@ -88,3 +124,5 @@ ee: - milestone: - events: - :push_event_payload + - iterations_cadences: + - :iterations diff --git a/lib/gitlab/import_export/group/legacy_tree_restorer.rb b/lib/gitlab/import_export/group/legacy_tree_restorer.rb index 8b39362b6bb..fa9e765b33a 100644 --- a/lib/gitlab/import_export/group/legacy_tree_restorer.rb +++ b/lib/gitlab/import_export/group/legacy_tree_restorer.rb @@ -68,23 +68,23 @@ module Gitlab def restorer @relation_tree_restorer ||= RelationTreeRestorer.new( - user: @user, - shared: @shared, - relation_reader: relation_reader, - members_mapper: members_mapper, - object_builder: object_builder, - relation_factory: relation_factory, - reader: reader, - importable: @group, + user: @user, + shared: @shared, + relation_reader: relation_reader, + members_mapper: members_mapper, + object_builder: object_builder, + relation_factory: relation_factory, + reader: reader, + importable: @group, importable_attributes: @group_attributes, - importable_path: nil + importable_path: nil ) end def create_group(group_hash:, parent_group:) group_params = { - name: group_hash['name'], - path: group_hash['path'], + name: group_hash['name'], + path: group_hash['path'], parent_id: parent_group&.id, visibility_level: sub_group_visibility_level(group_hash, parent_group) } diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 258078d595b..1b8436c4ed9 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -5,10 +5,11 @@ module Gitlab module Group class RelationFactory < Base::RelationFactory OVERRIDES = { - labels: :group_labels, + labels: :group_labels, priorities: :label_priorities, - label: :group_label, - parent: :epic + label: :group_label, + parent: :epic, + iterations_cadences: 'Iterations::Cadence' }.freeze EXISTING_OBJECT_RELATIONS = %i[ @@ -25,7 +26,10 @@ module Gitlab private def setup_models - setup_note if @relation_name == :notes + case @relation_name + when :notes then setup_note + when :'Iterations::Cadence' then setup_iterations_cadence + end update_group_references end @@ -44,6 +48,10 @@ module Gitlab def use_attributes_permitter? false end + + def setup_iterations_cadence + @relation_hash['automatic'] = false + end end end end diff --git a/lib/gitlab/import_export/group/relation_tree_restorer.rb b/lib/gitlab/import_export/group/relation_tree_restorer.rb index fab677bd772..5a78f2fb531 100644 --- a/lib/gitlab/import_export/group/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/group/relation_tree_restorer.rb @@ -136,9 +136,9 @@ module Gitlab attributes_permitter.permit(importable_class_sym, params) else Gitlab::ImportExport::AttributeCleaner.clean( - relation_hash: params, + relation_hash: params, relation_class: importable_class, - excluded_keys: excluded_keys_for_relation(importable_class_sym)) + excluded_keys: excluded_keys_for_relation(importable_class_sym)) end end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 796b9258e57..b4c86c3fc7f 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -46,7 +46,8 @@ module Gitlab group, group_tree, json_writer, - exportable_path: "groups/#{group.id}" + exportable_path: "groups/#{group.id}", + current_user: @current_user ).execute end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 78f43f79072..99396d64779 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -6,11 +6,7 @@ module Gitlab class StreamingSerializer include Gitlab::ImportExport::CommandLineUtil - BATCH_SIZE = 2 - - def self.batch_size(exportable) - BATCH_SIZE - end + BATCH_SIZE = 100 class Raw < String def to_json(*_args) @@ -18,8 +14,9 @@ module Gitlab end end - def initialize(exportable, relations_schema, json_writer, exportable_path:, logger: Gitlab::Export::Logger) + def initialize(exportable, relations_schema, json_writer, current_user:, exportable_path:, logger: Gitlab::Export::Logger) @exportable = exportable + @current_user = current_user @exportable_path = exportable_path @relations_schema = relations_schema @json_writer = json_writer @@ -63,7 +60,7 @@ module Gitlab private - attr_reader :json_writer, :relations_schema, :exportable, :logger + attr_reader :json_writer, :relations_schema, :exportable, :logger, :current_user def serialize_many_relations(key, records, options) log_relation_export(key, records.size) @@ -77,7 +74,7 @@ module Gitlab batch.each do |record| before_read_callback(record) - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) after_read_callback(record) end @@ -87,8 +84,29 @@ module Gitlab json_writer.write_relation_array(@exportable_path, key, enumerator) end + def exportable_json_record(record, options, key) + associations = relations_schema[:include_if_exportable]&.dig(key) + return Raw.new(record.to_json(options)) unless associations && options[:include] + + filtered_options = options.deep_dup + associations.each do |association| + filtered_options[:include].delete_if do |option| + !exportable_json_association?(option, record, association.to_sym) + end + end + + Raw.new(record.to_json(filtered_options)) + end + + def exportable_json_association?(option, record, association) + return true unless option.has_key?(association) + return false unless record.respond_to?(:exportable_association?) + + record.exportable_association?(association, current_user: current_user) + end + def batch(relation, key) - opts = { of: batch_size } + opts = { of: BATCH_SIZE } order_by = reorders(relation, key) # we need to sort issues by non primary key column(relative_position) @@ -115,7 +133,7 @@ module Gitlab enumerator = Enumerator.new do |items| records.each do |record| - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) end end @@ -125,7 +143,7 @@ module Gitlab def serialize_single_relation(key, record, options) log_relation_export(key) - json = Raw.new(record.to_json(options)) + json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) end @@ -138,10 +156,6 @@ module Gitlab relations_schema[:preload] end - def batch_size - @batch_size ||= self.class.batch_size(@exportable) - end - def reorders(relation, key) export_reorder = relations_schema[:export_reorder]&.dig(key) return unless export_reorder diff --git a/lib/gitlab/import_export/legacy_relation_tree_saver.rb b/lib/gitlab/import_export/legacy_relation_tree_saver.rb index c6b961ea210..cf75a2c7fa8 100644 --- a/lib/gitlab/import_export/legacy_relation_tree_saver.rb +++ b/lib/gitlab/import_export/legacy_relation_tree_saver.rb @@ -7,7 +7,7 @@ module Gitlab def serialize(exportable, relations_tree) Gitlab::ImportExport::FastHashSerializer - .new(exportable, relations_tree, batch_size: batch_size(exportable)) + .new(exportable, relations_tree) .execute end @@ -18,12 +18,6 @@ module Gitlab File.write(File.join(dir_path, filename), tree_json) end - - private - - def batch_size(exportable) - Gitlab::ImportExport::Json::StreamingSerializer.batch_size(exportable) - end end end end diff --git a/lib/gitlab/import_export/members_mapper.rb b/lib/gitlab/import_export/members_mapper.rb index b1f2a17d4b7..c94549a2b3f 100644 --- a/lib/gitlab/import_export/members_mapper.rb +++ b/lib/gitlab/import_export/members_mapper.rb @@ -139,7 +139,7 @@ module Gitlab end def parsed_hash(member) - Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, + Gitlab::ImportExport::AttributeCleaner.clean(relation_hash: member.deep_stringify_keys, relation_class: relation_class) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index c5b8f3fd35b..33e4823f192 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -29,6 +29,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - designs: - notes: - :author @@ -82,6 +85,9 @@ tree: - resource_label_events: - label: - :priorities + - resource_milestone_events: + - :milestone + - :resource_state_events - :external_pull_requests - ci_pipelines: - notes: @@ -287,6 +293,7 @@ included_attributes: - :forking_access_level - :metrics_dashboard_access_level - :operations_access_level + - :monitor_access_level - :analytics_access_level - :security_and_compliance_access_level - :container_registry_access_level @@ -551,6 +558,7 @@ included_attributes: - :failure_reason - :scheduled_at - :scheduling_type + - :ci_stage ci_pipelines: - :ref - :sha @@ -599,7 +607,6 @@ included_attributes: merge_request_assignees: - :user_id - :created_at - - :state merge_request_reviewers: - :user_id - :created_at @@ -699,6 +706,7 @@ included_attributes: - :metrics_dashboard_access_level - :analytics_access_level - :operations_access_level + - :monitor_access_level - :security_and_compliance_access_level - :container_registry_access_level - :package_registry_access_level @@ -721,6 +729,18 @@ included_attributes: - :build_git_strategy - :build_enabled - :security_and_compliance_enabled + resource_milestone_events: + - :user_id + - :action + - :created_at + - :state + resource_state_events: + - :user_id + - :state + - :created_at + - :source_commit + - :close_after_error_tracking_resolve + - :close_auto_resolve_prometheus_alert # Do not include the following attributes for the models specified. excluded_attributes: @@ -989,6 +1009,46 @@ excluded_attributes: milestone_releases: - :milestone_id - :release_id + resource_milestone_events: + - :id + - :issue_id + - :merge_request_id + - :milestone_id + resource_state_events: + - :id + - :issue_id + - :merge_request_id + - :epic_id + - :source_merge_request_id + iteration: + - :id + - :title + - :title_html + - :project_id + - :description_html + - :cached_markdown_version + - :iterations_cadence_id + - :sequence + resource_iteration_events: + - :id + - :issue_id + - :merge_request_id + - :iteration_id + iterations_cadence: + - :id + - :last_run_date + - :duration_in_weeks + - :iterations_in_advance + - :automatic + - :group_id + - :created_at + - :updated_at + - :start_date + - :active + - :roll_over + - :description + - :sequence + methods: notes: - :type @@ -1062,6 +1122,11 @@ ee: - epic_issue: - :epic - :issuable_sla + - iteration: + - :iterations_cadence + - resource_iteration_events: + - iteration: + - :iterations_cadence - protected_branches: - :unprotect_access_levels - protected_environments: @@ -1120,5 +1185,44 @@ ee: - :auto_fix_dependency_scanning - :auto_fix_sast project: - - :requirements_enabled - - :requirements_access_level + - :requirements_enabled + - :requirements_access_level + resource_iteration_events: + - :user_id + - :action + - :created_at + iteration: + - :iid + - :created_at + - :updated_at + - :start_date + - :due_date + - :group_id + - :description + iterations_cadence: + - :title + + preloads: + issues: + epic: + + # When associated resources are from outside the project, you might need to + # validate that a user who is exporting the project or group can access these + # associations. `include_if_exportable` accepts an array of associations for a + # resource. During export, the `exportable_association?` method on the + # resource is called with the association's name and user to validate if + # associated resource can be included in the export. + # + # This definition will call issue's `exportable_association?(:epic_issue, + # current_user: current_user)` method and include issue's epic_issue association + # for each issue only if the method returns true: + # + # Example: + # include_if_exportable: + # project: + # issues: + # - epic_issue + include_if_exportable: + project: + issues: + - :epic_issue diff --git a/lib/gitlab/import_export/project/import_task.rb b/lib/gitlab/import_export/project/import_task.rb index 59bb8af750e..89f2b36ea58 100644 --- a/lib/gitlab/import_export/project/import_task.rb +++ b/lib/gitlab/import_export/project/import_task.rb @@ -80,8 +80,8 @@ module Gitlab def import_params { namespace_id: namespace.id, - path: project_path, - file: File.open(file_path) + path: project_path, + file: File.open(file_path) } end diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index bf60d115a25..50a67a746f8 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -21,7 +21,7 @@ module Gitlab end def find - return if epic? && group.nil? + return if group_relation_without_group? return find_diff_commit_user if diff_commit_user? return find_diff_commit if diff_commit? @@ -60,7 +60,7 @@ module Gitlab def prepare_attributes attributes.dup.tap do |atts| - atts.delete('group') unless epic? + atts.delete('group') unless epic? || iteration? if label? atts['type'] = 'ProjectLabel' # Always create project labels @@ -141,6 +141,10 @@ module Gitlab klass == MergeRequestDiffCommit end + def iteration? + klass == Iteration + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -157,7 +161,13 @@ module Gitlab milestone.ensure_project_iid! milestone.save! end + + def group_relation_without_group? + (epic? || iteration?) && group.nil? + end end end end end + +Gitlab::ImportExport::Project::ObjectBuilder.prepend_mod diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index b40827e36f8..8e91adac196 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,8 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project' + exportable_path: 'project', + current_user: nil ) end diff --git a/lib/gitlab/import_export/project/relation_tree_restorer.rb b/lib/gitlab/import_export/project/relation_tree_restorer.rb index 6e9548f393a..47196db6f8a 100644 --- a/lib/gitlab/import_export/project/relation_tree_restorer.rb +++ b/lib/gitlab/import_export/project/relation_tree_restorer.rb @@ -5,7 +5,7 @@ module Gitlab module Project class RelationTreeRestorer < ImportExport::Group::RelationTreeRestorer # Relations which cannot be saved at project level (and have a group assigned) - GROUP_MODELS = [GroupLabel, Milestone, Epic].freeze + GROUP_MODELS = [GroupLabel, Milestone, Epic, Iteration].freeze private diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1b54e4b975e..bd34cd3ff6e 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -50,7 +50,8 @@ module Gitlab reader.project_tree, json_writer, exportable_path: "project", - logger: @logger + logger: @logger, + current_user: @current_user ) Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do diff --git a/lib/gitlab/instrumentation/redis.rb b/lib/gitlab/instrumentation/redis.rb index 4fee779c767..a371930621d 100644 --- a/lib/gitlab/instrumentation/redis.rb +++ b/lib/gitlab/instrumentation/redis.rb @@ -4,15 +4,20 @@ module Gitlab module Instrumentation # Aggregates Redis measurements from different request storage sources. class Redis + # Actioncable has it's separate instrumentation, but isn't configurable + # in the same way as all the other instances using a class. ActionCable = Class.new(RedisBase) - Cache = Class.new(RedisBase).enable_redis_cluster_validation - Queues = Class.new(RedisBase) - SharedState = Class.new(RedisBase).enable_redis_cluster_validation - TraceChunks = Class.new(RedisBase).enable_redis_cluster_validation - RateLimiting = Class.new(RedisBase).enable_redis_cluster_validation - Sessions = Class.new(RedisBase).enable_redis_cluster_validation - - STORAGES = [ActionCable, Cache, Queues, SharedState, TraceChunks, RateLimiting, Sessions].freeze + + STORAGES = ( + Gitlab::Redis::ALL_CLASSES.map do |redis_instance_class| + instrumentation_class = Class.new(RedisBase) + + instrumentation_class.enable_redis_cluster_validation unless redis_instance_class == Gitlab::Redis::Queues + + const_set(redis_instance_class.store_name, instrumentation_class) + instrumentation_class + end << ActionCable + ).freeze # Milliseconds represented in seconds (from 1 millisecond to 2 seconds). QUERY_TIME_BUCKETS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2].freeze diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 0beab008f73..0bd10597f24 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -20,21 +20,19 @@ module Gitlab ::RequestStore[call_duration_key] += duration end - def add_call_details(duration, args) + def add_call_details(duration, commands) return unless Gitlab::PerformanceBar.enabled_for_request? - # redis-rb passes an array (e.g. [[:get, key]]) - return unless args.length == 1 detail_store << { - cmd: args.first, + commands: commands, duration: duration, backtrace: ::Gitlab::BacktraceCleaner.clean_backtrace(caller) } end - def increment_request_count + def increment_request_count(amount = 1) ::RequestStore[request_count_key] ||= 0 - ::RequestStore[request_count_key] += 1 + ::RequestStore[request_count_key] += amount end def increment_read_bytes(num_bytes) @@ -78,9 +76,9 @@ module Gitlab self end - def instance_count_request + def instance_count_request(amount = 1) @request_counter ||= Gitlab::Metrics.counter(:gitlab_redis_client_requests_total, 'Client side Redis request count, per Redis server') - @request_counter.increment({ storage: storage_key }) + @request_counter.increment({ storage: storage_key }, amount) end def instance_count_exception(ex) diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 14474693ddf..7e2acb91b94 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -13,27 +13,15 @@ module Gitlab end end - def call(*args, &block) - start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined - instrumentation_class.instance_count_request - instrumentation_class.redis_cluster_validate!(args.first) - - super(*args, &block) - rescue ::Redis::BaseError => ex - instrumentation_class.instance_count_exception(ex) - raise ex - ensure - duration = Gitlab::Metrics::System.monotonic_time - start - - unless APDEX_EXCLUDE.include?(command_from_args(args)) - instrumentation_class.instance_observe_duration(duration) + def call(command) + instrument_call([command]) do + super end + end - if ::RequestStore.active? - # These metrics measure total Redis usage per Rails request / job. - instrumentation_class.increment_request_count - instrumentation_class.add_duration(duration) - instrumentation_class.add_call_details(duration, args) + def call_pipeline(pipeline) + instrument_call(pipeline.commands) do + super end end @@ -50,6 +38,31 @@ module Gitlab private + def instrument_call(commands) + start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + instrumentation_class.instance_count_request(commands.size) + + commands.each { |c| instrumentation_class.redis_cluster_validate!(c) } + + yield + rescue ::Redis::BaseError => ex + instrumentation_class.instance_count_exception(ex) + raise ex + ensure + duration = Gitlab::Metrics::System.monotonic_time - start + + unless exclude_from_apdex?(commands) + commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } + end + + if ::RequestStore.active? + # These metrics measure total Redis usage per Rails request / job. + instrumentation_class.increment_request_count(commands.size) + instrumentation_class.add_duration(duration) + instrumentation_class.add_call_details(duration, commands) + end + end + def measure_write_size(command) size = 0 @@ -97,10 +110,8 @@ module Gitlab @options[:instrumentation_class] # rubocop:disable Gitlab/ModuleWithInstanceVariables end - def command_from_args(args) - command = args[0] - command = command[0] if command.is_a?(Array) - command.to_s.downcase + def exclude_from_apdex?(commands) + commands.any? { |command| APDEX_EXCLUDE.include?(command.first.to_s.downcase) } end end end diff --git a/lib/gitlab/issuable/clone/copy_resource_events_service.rb b/lib/gitlab/issuable/clone/copy_resource_events_service.rb index 563805fcb01..448ac4c2ae0 100644 --- a/lib/gitlab/issuable/clone/copy_resource_events_service.rb +++ b/lib/gitlab/issuable/clone/copy_resource_events_service.rb @@ -49,7 +49,7 @@ module Gitlab event.attributes .except(*blocked_state_event_attributes) .merge(entity_key => new_entity.id, - 'state' => ResourceStateEvent.states[event.state]) + 'state' => ResourceStateEvent.states[event.state]) end end @@ -62,9 +62,9 @@ module Gitlab event.attributes .except('id') .merge(entity_key => new_entity.id, - 'milestone_id' => milestone&.id, - 'action' => ResourceMilestoneEvent.actions[event.action], - 'state' => ResourceMilestoneEvent.states[event.state]) + 'milestone_id' => milestone&.id, + 'action' => ResourceMilestoneEvent.actions[event.action], + 'state' => ResourceMilestoneEvent.states[event.state]) end def copy_events(table_name, events_to_copy) diff --git a/lib/gitlab/jira_import.rb b/lib/gitlab/jira_import.rb index 60344e4be68..fd41d9eeb5a 100644 --- a/lib/gitlab/jira_import.rb +++ b/lib/gitlab/jira_import.rb @@ -66,11 +66,6 @@ module Gitlab cache_class.write(cache_key, value) end - def self.cache_issue_mapping(issue_id, jira_issue_id, project_id) - cache_key = JiraImport.jira_item_cache_key(project_id, jira_issue_id, :issues) - cache_class.write(cache_key, issue_id) - end - def self.get_import_label_id(project_id) cache_class.read(JiraImport.import_label_cache_key(project_id)) end diff --git a/lib/gitlab/kubernetes.rb b/lib/gitlab/kubernetes.rb index 15163bd4a57..0a0a1defd11 100644 --- a/lib/gitlab/kubernetes.rb +++ b/lib/gitlab/kubernetes.rb @@ -71,11 +71,11 @@ module Gitlab containers.map do |container| { - selectors: { pod: pod_name, container: container["name"] }, - url: container_exec_url(api_url, namespace, pod_name, container["name"]), + selectors: { pod: pod_name, container: container["name"] }, + url: container_exec_url(api_url, namespace, pod_name, container["name"]), subprotocols: ['channel.k8s.io'], - headers: ::Gitlab::Kubernetes.build_header_hash, - created_at: created_at + headers: ::Gitlab::Kubernetes.build_header_hash, + created_at: created_at } end end diff --git a/lib/gitlab/legacy_github_import/client.rb b/lib/gitlab/legacy_github_import/client.rb index 7d78c8dee25..dd1502bbbcd 100644 --- a/lib/gitlab/legacy_github_import/client.rb +++ b/lib/gitlab/legacy_github_import/client.rb @@ -79,6 +79,20 @@ module Gitlab @users[login] = api.user(login) end + def repository(id) + request(:repository, id).to_h + end + + def repos + repositories = request(:repos, nil) + + if repositories.is_a?(Array) + repositories.map(&:to_h) + else + repositories + end + end + private def api_endpoint diff --git a/lib/gitlab/legacy_github_import/project_creator.rb b/lib/gitlab/legacy_github_import/project_creator.rb index c54325bcdf5..01e04fa9c81 100644 --- a/lib/gitlab/legacy_github_import/project_creator.rb +++ b/lib/gitlab/legacy_github_import/project_creator.rb @@ -18,11 +18,11 @@ module Gitlab attrs = { name: name, path: name, - description: repo.description, + description: repo[:description], namespace_id: namespace.id, visibility_level: visibility_level, import_type: type, - import_source: repo.full_name, + import_source: repo[:full_name], import_url: import_url, skip_wiki: skip_wiki }.merge!(extra_attrs) @@ -33,11 +33,11 @@ module Gitlab private def import_url - repo.clone_url.sub('://', "://#{session_data[:github_access_token]}@") + repo[:clone_url].sub('://', "://#{session_data[:github_access_token]}@") end def visibility_level - visibility_level = repo.private ? Gitlab::VisibilityLevel::PRIVATE : @namespace.visibility_level + visibility_level = repo[:private] ? Gitlab::VisibilityLevel::PRIVATE : @namespace.visibility_level visibility_level = Gitlab::CurrentSettings.default_project_visibility if Gitlab::CurrentSettings.restricted_visibility_levels.include?(visibility_level) visibility_level @@ -49,7 +49,7 @@ module Gitlab # repository already exist. # def skip_wiki - repo.has_wiki? + repo[:has_wiki] end end end diff --git a/lib/gitlab/mailgun/webhook_processors/failure_logger.rb b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb index a7a85bd1672..fa72abf1311 100644 --- a/lib/gitlab/mailgun/webhook_processors/failure_logger.rb +++ b/lib/gitlab/mailgun/webhook_processors/failure_logger.rb @@ -5,11 +5,12 @@ module Gitlab module WebhookProcessors class FailureLogger < Base def execute - log_failure if permanent_failure? || temporary_failure_over_threshold? + log_failure if permanent_failure_over_threshold? || temporary_failure_over_threshold? end - def permanent_failure? - payload['event'] == 'failed' && payload['severity'] == 'permanent' + def permanent_failure_over_threshold? + payload['event'] == 'failed' && payload['severity'] == 'permanent' && + Gitlab::ApplicationRateLimiter.throttled?(:permanent_email_failure, scope: payload['recipient']) end def temporary_failure_over_threshold? diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 80dff075391..6fe9bb10cdf 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -14,9 +14,9 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - redis.multi do - redis.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - redis.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + redis.multi do |multi| + multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) end end end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index f635f41ec39..aab58bfa211 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -31,7 +31,7 @@ module Gitlab if job.is_a?(ActionMailer::MailDeliveryJob) { "class" => job.arguments.first, - "jid" => job.job_id + "jid" => job.job_id } else job diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 5a8efa34097..752ab153f37 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -10,9 +10,9 @@ module Gitlab results = {} Gitlab::Redis::Cache.with do |r| - r.pipelined do + r.pipelined do |pipeline| subjects.each do |subject| - results[subject.cache_key] = new(subject).read + results[subject.cache_key] = new(subject).read(pipeline) end end end @@ -34,11 +34,15 @@ module Gitlab end end - def read + def read(pipeline = nil) @loaded = true - Gitlab::Redis::Cache.with do |r| - r.mapped_hmget(markdown_cache_key, *fields) + if pipeline + pipeline.mapped_hmget(markdown_cache_key, *fields) + else + Gitlab::Redis::Cache.with do |r| + r.mapped_hmget(markdown_cache_key, *fields) + end end end diff --git a/lib/gitlab/memory/jemalloc.rb b/lib/gitlab/memory/jemalloc.rb index 7163a70a5cb..e20e186cab9 100644 --- a/lib/gitlab/memory/jemalloc.rb +++ b/lib/gitlab/memory/jemalloc.rb @@ -27,21 +27,27 @@ module Gitlab # Write jemalloc stats to the given directory # @param [String] path Directory path the dump will be put into + # @param [String] tmp_dir Directory path the dump will be streaming to. It is moved to `path` when finished. # @param [String] format `json` or `txt` # @param [String] filename_label Optional custom string that will be injected into the file name, e.g. `worker_0` # @return [String] Full path to the resulting dump file - def dump_stats(path:, format: STATS_DEFAULT_FORMAT, filename_label: nil) + def dump_stats(path:, tmp_dir: Dir.tmpdir, format: STATS_DEFAULT_FORMAT, filename_label: nil) verify_format!(format) format_settings = STATS_FORMATS[format] + tmp_file_path = File.join(tmp_dir, file_name(format_settings[:extension], filename_label)) file_path = File.join(path, file_name(format_settings[:extension], filename_label)) with_malloc_stats_print do |stats_print| - File.open(file_path, 'wb') do |io| + File.open(tmp_file_path, 'wb') do |io| write_stats(stats_print, io, format_settings) end end + # On OSX, `with_malloc_stats_print` is no-op, and, as result, no file will be written + return unless File.exist?(tmp_file_path) + + FileUtils.mv(tmp_file_path, file_path) file_path end diff --git a/lib/gitlab/memory/reports/jemalloc_stats.rb b/lib/gitlab/memory/reports/jemalloc_stats.rb index b99bec4ac3e..05f0717d7c3 100644 --- a/lib/gitlab/memory/reports/jemalloc_stats.rb +++ b/lib/gitlab/memory/reports/jemalloc_stats.rb @@ -18,12 +18,19 @@ module Gitlab def initialize(reports_path:) @reports_path = reports_path + + # Store report in tmp subdir while it is still streaming. + # This will clearly separate finished reports from the files we are still writing to. + @tmp_dir = File.join(@reports_path, 'tmp') + FileUtils.mkdir_p(@tmp_dir) end def run return unless active? - Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, filename_label: worker_id).tap { cleanup } + Gitlab::Memory::Jemalloc.dump_stats(path: reports_path, tmp_dir: @tmp_dir, filename_label: worker_id).tap do + cleanup + end end def active? diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 91edb68ad66..38231fa933b 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -16,8 +16,9 @@ module Gitlab # The duration for which a process may be above a given fragmentation # threshold is computed as `max_strikes * sleep_time_seconds`. class Watchdog - DEFAULT_SLEEP_TIME_SECONDS = 60 - DEFAULT_HEAP_FRAG_THRESHOLD = 0.5 + DEFAULT_SLEEP_TIME_SECONDS = 60 * 5 + DEFAULT_MAX_HEAP_FRAG = 0.5 + DEFAULT_MAX_MEM_GROWTH = 3.0 DEFAULT_MAX_STRIKES = 5 # This handler does nothing. It returns `false` to indicate to the @@ -29,7 +30,7 @@ module Gitlab class NullHandler include Singleton - def on_high_heap_fragmentation(value) + def call # NOP false end @@ -41,7 +42,7 @@ module Gitlab @pid = pid end - def on_high_heap_fragmentation(value) + def call Process.kill(:TERM, @pid) true end @@ -55,7 +56,7 @@ module Gitlab @worker = ::Puma::Cluster::WorkerHandle.new(0, $$, 0, puma_options) end - def on_high_heap_fragmentation(value) + def call @worker.term true end @@ -63,6 +64,9 @@ module Gitlab # max_heap_fragmentation: # The degree to which the Ruby heap is allowed to be fragmented. Range [0,1]. + # max_mem_growth: + # A multiplier for how much excess private memory a worker can map compared to a reference process + # (itself or the primary in a pre-fork server.) # max_strikes: # How many times the process is allowed to be above max_heap_fragmentation before # a handler is invoked. @@ -71,7 +75,8 @@ module Gitlab def initialize( handler: NullHandler.instance, logger: Logger.new($stdout), - max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_HEAP_FRAG_THRESHOLD, + max_heap_fragmentation: ENV['GITLAB_MEMWD_MAX_HEAP_FRAG']&.to_f || DEFAULT_MAX_HEAP_FRAG, + max_mem_growth: ENV['GITLAB_MEMWD_MAX_MEM_GROWTH']&.to_f || DEFAULT_MAX_MEM_GROWTH, max_strikes: ENV['GITLAB_MEMWD_MAX_STRIKES']&.to_i || DEFAULT_MAX_STRIKES, sleep_time_seconds: ENV['GITLAB_MEMWD_SLEEP_TIME_SEC']&.to_i || DEFAULT_SLEEP_TIME_SECONDS, **options) @@ -79,17 +84,37 @@ module Gitlab @handler = handler @logger = logger - @max_heap_fragmentation = max_heap_fragmentation @sleep_time_seconds = sleep_time_seconds @max_strikes = max_strikes + @stats = { + heap_frag: { + max: max_heap_fragmentation, + strikes: 0 + }, + mem_growth: { + max: max_mem_growth, + strikes: 0 + } + } @alive = true - @strikes = 0 init_prometheus_metrics(max_heap_fragmentation) end - attr_reader :strikes, :max_heap_fragmentation, :max_strikes, :sleep_time_seconds + attr_reader :max_strikes, :sleep_time_seconds + + def max_heap_fragmentation + @stats[:heap_frag][:max] + end + + def max_mem_growth + @stats[:mem_growth][:max] + end + + def strikes(stat) + @stats[stat][:strikes] + end def call @logger.info(log_labels.merge(message: 'started')) @@ -97,7 +122,10 @@ module Gitlab while @alive sleep(@sleep_time_seconds) - monitor_heap_fragmentation if Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + next unless Feature.enabled?(:gitlab_memory_watchdog, type: :ops) + + monitor_heap_fragmentation + monitor_memory_growth end @logger.info(log_labels.merge(message: 'stopped')) @@ -109,32 +137,73 @@ module Gitlab private - def monitor_heap_fragmentation - heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + def monitor_memory_condition(stat_key) + return unless @alive + + stat = @stats[stat_key] + + ok, labels = yield(stat) - if heap_fragmentation > @max_heap_fragmentation - @strikes += 1 - @heap_frag_violations.increment + if ok + stat[:strikes] = 0 else - @strikes = 0 + stat[:strikes] += 1 + @counter_violations.increment(reason: stat_key.to_s) end - if @strikes > @max_strikes - # If the handler returns true, it means the event is handled and we can shut down. - @alive = !handle_heap_fragmentation_limit_exceeded(heap_fragmentation) - @strikes = 0 + if stat[:strikes] > @max_strikes + @alive = !memory_limit_exceeded_callback(stat_key, labels) + stat[:strikes] = 0 end end - def handle_heap_fragmentation_limit_exceeded(value) - @logger.warn( - log_labels.merge( - message: 'heap fragmentation limit exceeded', - memwd_cur_heap_frag: value - )) - @heap_frag_violations_handled.increment + def monitor_heap_fragmentation + monitor_memory_condition(:heap_frag) do |stat| + heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation + [ + heap_fragmentation <= stat[:max], + { + message: 'heap fragmentation limit exceeded', + memwd_cur_heap_frag: heap_fragmentation, + memwd_max_heap_frag: stat[:max] + } + ] + end + end + + def monitor_memory_growth + monitor_memory_condition(:mem_growth) do |stat| + worker_uss = Gitlab::Metrics::System.memory_usage_uss_pss[:uss] + reference_uss = reference_mem[:uss] + memory_limit = stat[:max] * reference_uss + [ + worker_uss <= memory_limit, + { + message: 'memory limit exceeded', + memwd_uss_bytes: worker_uss, + memwd_ref_uss_bytes: reference_uss, + memwd_max_uss_bytes: memory_limit + } + ] + end + end + + # On pre-fork systems this would be the primary process memory from which workers fork. + # Otherwise it is the current process' memory. + # + # We initialize this lazily because in the initializer the application may not have + # finished booting yet, which would yield an incorrect baseline. + def reference_mem + @reference_mem ||= Gitlab::Metrics::System.memory_usage_uss_pss(pid: Gitlab::Cluster::PRIMARY_PID) + end + + def memory_limit_exceeded_callback(stat_key, handler_labels) + all_labels = log_labels.merge(handler_labels) + .merge(memwd_cur_strikes: strikes(stat_key)) + @logger.warn(all_labels) + @counter_violations_handled.increment(reason: stat_key.to_s) - handler.on_high_heap_fragmentation(value) + handler.call end def handler @@ -151,9 +220,7 @@ module Gitlab worker_id: worker_id, memwd_handler_class: handler.class.name, memwd_sleep_time_s: @sleep_time_seconds, - memwd_max_heap_frag: @max_heap_fragmentation, memwd_max_strikes: @max_strikes, - memwd_cur_strikes: @strikes, memwd_rss_bytes: process_rss_bytes } end @@ -174,14 +241,14 @@ module Gitlab @heap_frag_limit.set({}, max_heap_fragmentation) default_labels = { pid: worker_id } - @heap_frag_violations = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_total, - 'Total number of times heap fragmentation in a Ruby process exceeded its allowed maximum', + @counter_violations = Gitlab::Metrics.counter( + :gitlab_memwd_violations_total, + 'Total number of times a Ruby process violated a memory threshold', default_labels ) - @heap_frag_violations_handled = Gitlab::Metrics.counter( - :gitlab_memwd_heap_frag_violations_handled_total, - 'Total number of times heap fragmentation violations in a Ruby process were handled', + @counter_violations_handled = Gitlab::Metrics.counter( + :gitlab_memwd_violations_handled_total, + 'Total number of times Ruby process memory violations were handled', default_labels ) end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index 55d14d6f94a..622b6adec7e 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -40,8 +40,8 @@ module Gitlab def formatted_panel { - title: panel[:title], - type: CHART_TYPE, + title: panel[:title], + type: CHART_TYPE, y_label: '', # Grafana panels do not include a Y-Axis label metrics: panel[:targets].map.with_index do |target, idx| formatted_metric(target, idx) @@ -51,9 +51,9 @@ module Gitlab def formatted_metric(metric, idx) { - id: "#{metric[:legendFormat]}_#{idx}", - query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]), + id: "#{metric[:legendFormat]}_#{idx}", + query_range: format_query(metric), + label: replace_variables(metric[:legendFormat]), prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) }.compact end diff --git a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb index 4e46eec17d6..3650ddf698a 100644 --- a/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb +++ b/lib/gitlab/metrics/dashboard/transformers/yml/v1/prometheus_metrics.rb @@ -24,15 +24,15 @@ module Gitlab panel_group[:panels].each do |panel| panel[:metrics].each do |metric| prometheus_metrics << { - project: project, - title: panel[:title], - y_label: panel[:y_label], - query: metric[:query_range] || metric[:query], - unit: metric[:unit], - legend: metric[:label], - identifier: metric[:id], - group: Enums::PrometheusMetric.groups[:custom], - common: false, + project: project, + title: panel[:title], + y_label: panel[:y_label], + query: metric[:query_range] || metric[:query], + unit: metric[:unit], + legend: metric[:label], + identifier: metric[:id], + group: Enums::PrometheusMetric.groups[:custom], + common: false, dashboard_path: dashboard_path }.compact end diff --git a/lib/gitlab/metrics/dashboard/validator/client.rb b/lib/gitlab/metrics/dashboard/validator/client.rb index 588c677ca28..29f1274a097 100644 --- a/lib/gitlab/metrics/dashboard/validator/client.rb +++ b/lib/gitlab/metrics/dashboard/validator/client.rb @@ -34,8 +34,8 @@ module Gitlab def post_schema_validator PostSchemaValidator.new( - project: project, - metric_ids: custom_formats.metric_ids_cache, + project: project, + metric_ids: custom_formats.metric_ids_cache, dashboard_path: dashboard_path ) end diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb index e17f1c13cf0..258b655229e 100644 --- a/lib/gitlab/metrics/exporter/metrics_middleware.rb +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -27,8 +27,8 @@ module Gitlab labels = { method: env['REQUEST_METHOD'].downcase, - path: env['PATH_INFO'].to_s, - code: response.first.to_s + path: env['PATH_INFO'].to_s, + code: response.first.to_s } @requests_total.increment(labels) diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb new file mode 100644 index 00000000000..e37129fed38 --- /dev/null +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module GlobalSearchSlis + class << self + # The following targets are the 99.95th percentile of code searches + # gathered on 24-08-2022 + # from https://log.gprd.gitlab.net/goto/0c89cd80-23af-11ed-8656-f5f2137823ba (internal only) + BASIC_CONTENT_TARGET_S = 7.031 + BASIC_CODE_TARGET_S = 21.903 + ADVANCED_CONTENT_TARGET_S = 4.865 + ADVANCED_CODE_TARGET_S = 13.546 + + def initialize_slis! + if Feature.enabled?(:global_search_custom_slis) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) + end + + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:global_search, possible_labels) + end + + def record_apdex(elapsed:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_custom_slis) + + Gitlab::Metrics::Sli::Apdex[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + success: elapsed < duration_target(search_type, search_scope) + ) + end + + def record_error_rate(error:, search_type:, search_level:, search_scope:) + return unless Feature.enabled?(:global_search_error_rate_sli) + + Gitlab::Metrics::Sli::ErrorRate[:global_search].increment( + labels: labels(search_type: search_type, search_level: search_level, search_scope: search_scope), + error: error + ) + end + + private + + def duration_target(search_type, search_scope) + if search_type == 'basic' && content_search?(search_scope) + BASIC_CONTENT_TARGET_S + elsif search_type == 'basic' && code_search?(search_scope) + BASIC_CODE_TARGET_S + elsif search_type == 'advanced' && content_search?(search_scope) + ADVANCED_CONTENT_TARGET_S + elsif search_type == 'advanced' && code_search?(search_scope) + ADVANCED_CODE_TARGET_S + end + end + + def search_types + %w[basic advanced] + end + + def search_levels + %w[project group global] + end + + def search_scopes + Gitlab::Search::AbuseDetection::ALLOWED_SCOPES + end + + def endpoint_ids + ['SearchController#show', 'GET /api/:version/search', 'GET /api/:version/projects/:id/(-/)search', + 'GET /api/:version/groups/:id/(-/)search'] + end + + def possible_labels + search_types.flat_map do |search_type| + search_levels.flat_map do |search_level| + search_scopes.flat_map do |search_scope| + endpoint_ids.flat_map do |endpoint_id| + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + end + end + end + end + + def labels(search_type:, search_level:, search_scope:) + { + search_type: search_type, + search_level: search_level, + search_scope: search_scope, + endpoint_id: endpoint_id + } + end + + def endpoint_id + ::Gitlab::ApplicationContext.current_context_attribute(:caller_id) + end + + def code_search?(search_scope) + search_scope == 'blobs' + end + + def content_search?(search_scope) + !code_search?(search_scope) + end + end + end + end +end diff --git a/lib/gitlab/metrics/samplers/puma_sampler.rb b/lib/gitlab/metrics/samplers/puma_sampler.rb index 848a55e59ff..d818aa43853 100644 --- a/lib/gitlab/metrics/samplers/puma_sampler.rb +++ b/lib/gitlab/metrics/samplers/puma_sampler.rb @@ -12,15 +12,15 @@ module Gitlab def init_metrics { - puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), - puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), - puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), - puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), + puma_workers: ::Gitlab::Metrics.gauge(:puma_workers, 'Total number of workers'), + puma_running_workers: ::Gitlab::Metrics.gauge(:puma_running_workers, 'Number of active workers'), + puma_stale_workers: ::Gitlab::Metrics.gauge(:puma_stale_workers, 'Number of stale workers'), + puma_running: ::Gitlab::Metrics.gauge(:puma_running, 'Number of running threads'), puma_queued_connections: ::Gitlab::Metrics.gauge(:puma_queued_connections, 'Number of connections in that worker\'s "todo" set waiting for a worker thread'), puma_active_connections: ::Gitlab::Metrics.gauge(:puma_active_connections, 'Number of threads processing a request'), - puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), - puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), - puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') + puma_pool_capacity: ::Gitlab::Metrics.gauge(:puma_pool_capacity, 'Number of requests the worker is capable of taking right now'), + puma_max_threads: ::Gitlab::Metrics.gauge(:puma_max_threads, 'Maximum number of worker threads'), + puma_idle_threads: ::Gitlab::Metrics.gauge(:puma_idle_threads, 'Number of spawned threads which are not processing a request') } end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 8e002293347..4fe338ffc7f 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -31,16 +31,16 @@ module Gitlab def init_metrics metrics = { - file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), - process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), - process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), - process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), - process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), + file_descriptors: ::Gitlab::Metrics.gauge(metric_name(:file, :descriptors), 'File descriptors used', labels), + process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), + process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), + process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), - process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), - sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), - gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), - heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) + process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), + sampler_duration: ::Gitlab::Metrics.counter(metric_name(:sampler, :duration_seconds_total), 'Sampler time', labels), + gc_duration_seconds: ::Gitlab::Metrics.histogram(metric_name(:gc, :duration_seconds), 'GC time', labels, GC_REPORT_BUCKETS), + heap_fragmentation: ::Gitlab::Metrics.gauge(metric_name(:gc_stat_ext, :heap_fragmentation), 'Ruby heap fragmentation', labels) } GC.stat.keys.each do |key| diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index e646846face..d7eef722d6e 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -10,8 +10,8 @@ module Gitlab extend self PROC_STAT_PATH = '/proc/self/stat' - PROC_STATUS_PATH = '/proc/self/status' - PROC_SMAPS_ROLLUP_PATH = '/proc/self/smaps_rollup' + PROC_STATUS_PATH = '/proc/%s/status' + PROC_SMAPS_ROLLUP_PATH = '/proc/%s/smaps_rollup' PROC_LIMITS_PATH = '/proc/self/limits' PROC_FD_GLOB = '/proc/self/fd/*' @@ -34,14 +34,14 @@ module Gitlab } end - # Returns the current process' RSS (resident set size) in bytes. - def memory_usage_rss - sum_matches(PROC_STATUS_PATH, rss: RSS_PATTERN)[:rss].kilobytes + # Returns the given process' RSS (resident set size) in bytes. + def memory_usage_rss(pid: 'self') + sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes end - # Returns the current process' USS/PSS (unique/proportional set size) in bytes. - def memory_usage_uss_pss - sum_matches(PROC_SMAPS_ROLLUP_PATH, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) + # Returns the given process' USS/PSS (unique/proportional set size) in bytes. + def memory_usage_uss_pss(pid: 'self') + sum_matches(PROC_SMAPS_ROLLUP_PATH % pid, uss: PRIVATE_PAGES_PATTERN, pss: PSS_PATTERN) .transform_values(&:kilobytes) end diff --git a/lib/gitlab/nav/top_nav_menu_builder.rb b/lib/gitlab/nav/top_nav_menu_builder.rb index 721ae1889b8..dca3432a6a1 100644 --- a/lib/gitlab/nav/top_nav_menu_builder.rb +++ b/lib/gitlab/nav/top_nav_menu_builder.rb @@ -6,9 +6,15 @@ module Gitlab def initialize @primary = [] @secondary = [] + @last_header_added = nil end - def add_primary_menu_item(**args) + def add_primary_menu_item(header: nil, **args) + if header && (header != @last_header_added) + add_menu_header(dest: @primary, title: header) + @last_header_added = header + end + add_menu_item(dest: @primary, **args) end @@ -30,6 +36,12 @@ module Gitlab dest.push(item) end + + def add_menu_header(dest:, **args) + header = ::Gitlab::Nav::TopNavMenuHeader.build(**args) + + dest.push(header) + end end end end diff --git a/lib/gitlab/nav/top_nav_menu_header.rb b/lib/gitlab/nav/top_nav_menu_header.rb new file mode 100644 index 00000000000..520091dbd97 --- /dev/null +++ b/lib/gitlab/nav/top_nav_menu_header.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Gitlab + module Nav + class TopNavMenuHeader + def self.build(title:) + { + type: :header, + title: title + } + end + end + end +end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index 4cb38e6bb9b..75eb0e8a264 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -11,6 +11,7 @@ module Gitlab def self.build(id:, title:, active: false, icon: '', href: '', view: '', css_class: nil, data: nil, emoji: nil) { id: id, + type: :item, title: title, active: active, icon: icon, diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index 11ca6a3a3ba..a8e25708107 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -42,11 +42,14 @@ module Gitlab def build menu = @menu_builder.build + hide_menu_text = Feature.enabled?(:new_navbar_layout) + menu.merge({ views: @views, shortcuts: @shortcuts, - activeTitle: _('Menu') - }) + menuTitle: (_('Menu') unless hide_menu_text), + menuTooltip: (_('Main menu') if hide_menu_text) + }.compact) end end end diff --git a/lib/gitlab/no_cache_headers.rb b/lib/gitlab/no_cache_headers.rb index f80ca2c1369..2d03741480d 100644 --- a/lib/gitlab/no_cache_headers.rb +++ b/lib/gitlab/no_cache_headers.rb @@ -4,8 +4,8 @@ module Gitlab module NoCacheHeaders DEFAULT_GITLAB_NO_CACHE_HEADERS = { 'Cache-Control' => "#{ActionDispatch::Http::Cache::Response::DEFAULT_CACHE_CONTROL}, no-store, no-cache", - 'Pragma' => 'no-cache', # HTTP 1.0 compatibility - 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' + 'Pragma' => 'no-cache', # HTTP 1.0 compatibility + 'Expires' => 'Fri, 01 Jan 1990 00:00:00 GMT' }.freeze def no_cache_headers diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index 1f1061fe4f1..d4de2791195 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -38,7 +38,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else @@ -52,7 +52,7 @@ module Gitlab if finder.is_a?(BranchesFinder) Feature.enabled?(:branch_list_keyset_pagination, project) elsif finder.is_a?(TagsFinder) - Feature.enabled?(:tag_list_keyset_pagination, project) + true elsif finder.is_a?(::Repositories::TreeFinder) Feature.enabled?(:repository_tree_gitaly_pagination, project) else diff --git a/lib/gitlab/pagination/keyset/column_order_definition.rb b/lib/gitlab/pagination/keyset/column_order_definition.rb index 302e7b406b1..d1fe1d2dfc1 100644 --- a/lib/gitlab/pagination/keyset/column_order_definition.rb +++ b/lib/gitlab/pagination/keyset/column_order_definition.rb @@ -213,7 +213,7 @@ module Gitlab attr_reader :reversed_order_expression, :nullable, :distinct def calculate_reversed_order(order_expression) - unless AREL_ORDER_CLASSES.has_key?(order_expression.class) # Arel can reverse simple orders + unless order_expression.is_a?(Arel::Nodes::Ordering) raise "Couldn't determine reversed order for `#{order_expression}`, please provide the `reversed_order_expression` parameter." end @@ -229,10 +229,10 @@ module Gitlab end def parse_order_direction(order_expression, order_direction) - transformed_order_direction = if order_direction.nil? && AREL_ORDER_CLASSES[order_expression.class] - AREL_ORDER_CLASSES[order_expression.class] - elsif order_direction.present? + transformed_order_direction = if order_direction.present? order_direction.to_s.downcase.to_sym + elsif order_expression.is_a?(Arel::Nodes::Ordering) + AREL_ORDER_CLASSES[order_expression.class] || AREL_ORDER_CLASSES[order_expression.value.class] end unless REVERSED_ORDER_DIRECTIONS.has_key?(transformed_order_direction) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb new file mode 100644 index 00000000000..630c364d455 --- /dev/null +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Patch to address https://github.com/ondrejbartas/sidekiq-cron/issues/361 +# This restores the poll interval to v1.2.0 behavior +# https://github.com/ondrejbartas/sidekiq-cron/blob/v1.2.0/lib/sidekiq/cron/poller.rb#L36-L38 +# This patch only applies to v1.4.0 +require 'sidekiq/cron/version' + +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.4.0') + raise 'New version of sidekiq-cron detected, please remove or update this patch' +end + +module Gitlab + module Patch + module SidekiqCronPoller + def poll_interval_average + Sidekiq.options[:poll_interval] || Sidekiq::Cron::POLL_INTERVAL + end + end + end +end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 189627506f3..4883c649a62 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -207,19 +207,22 @@ module Gitlab desc { _('Add Zoom meeting') } explanation { _('Adds a Zoom meeting.') } - params '<Zoom URL>' + params do + zoom_link_params + end types Issue condition do @zoom_service = zoom_link_service + @zoom_service.can_add_link? end - parse_params do |link| - @zoom_service.parse_link(link) + parse_params do |link_params| + @zoom_service.parse_link(link_params) end - command :zoom do |link| - result = @zoom_service.add_link(link) + command :zoom do |link, link_text = nil| + result = add_zoom_link(link, link_text) @execution_message[:zoom] = result.message - @updates.merge!(result.payload) if result.payload + merge_updates(result, @updates) end desc { _('Remove Zoom meeting') } @@ -315,12 +318,52 @@ module Gitlab @updates[:remove_contacts] = contact_emails.split(' ') end - private - - def zoom_link_service - ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + desc { _('Add a timeline event to incident') } + explanation { _('Adds a timeline event to incident.') } + params '<timeline comment> | <date(YYYY-MM-DD)> <time(HH:MM)>' + types Issue + condition do + quick_action_target.incident? && + current_user.can?(:admin_incident_management_timeline_event, quick_action_target) + end + parse_params do |event_params| + Gitlab::QuickActions::TimelineTextAndDateTimeSeparator.new(event_params).execute + end + command :timeline do |event_text, date_time| + if event_text && date_time + timeline_event = timeline_event_create_service(event_text, date_time).execute + + @execution_message[:timeline] = + if timeline_event.success? + _('Timeline event added successfully.') + else + _('Something went wrong while adding timeline event.') + end + end end end + + private + + def zoom_link_service + ::Issues::ZoomLinkService.new(project: quick_action_target.project, current_user: current_user, params: { issue: quick_action_target }) + end + + def zoom_link_params + '<Zoom URL>' + end + + def add_zoom_link(link, _link_text) + zoom_link_service.add_link(link) + end + + def merge_updates(result, update_hash) + update_hash.merge!(result.payload) if result.payload + end + + def timeline_event_create_service(event_text, event_date_time) + ::IncidentManagement::TimelineEvents::CreateService.new(quick_action_target, current_user, { note: event_text, occurred_at: event_date_time, editable: true }) + end end end end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 3cb01db1491..d38b81bff0b 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -88,33 +88,21 @@ module Gitlab @execution_message[:rebase] = _('Scheduled a rebase of branch %{branch}.') % { branch: branch } end - desc { _('Toggle the Draft status') } + desc { _('Set the Draft status') } explanation do - noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.draft? - _("Marks this %{noun} as ready.") - else - _("Marks this %{noun} as a draft.") - end % { noun: noun } + draft_action_message(_("Marks")) end execution_message do - noun = quick_action_target.to_ability_name.humanize(capitalize: false) - if quick_action_target.draft? - _("Marked this %{noun} as ready.") - else - _("Marked this %{noun} as a draft.") - end % { noun: noun } + draft_action_message(_("Marked")) end types MergeRequest condition do quick_action_target.respond_to?(:draft?) && - # Allow it to mark as draft on MR creation page or through MR notes - # (quick_action_target.new_record? || current_user.can?(:"update_#{quick_action_target.to_ability_name}", quick_action_target)) end command :draft do - @updates[:wip_event] = quick_action_target.draft? ? 'ready' : 'draft' + @updates[:wip_event] = draft_action_command end desc { _('Set the Ready status') } @@ -317,6 +305,25 @@ module Gitlab end end + def draft_action_message(verb) + noun = quick_action_target.to_ability_name.humanize(capitalize: false) + if !quick_action_target.draft? + _("%{verb} this %{noun} as a draft.") + elsif Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) + _("%{verb} this %{noun} as ready.") + else + _("No change to this %{noun}'s draft status.") + end % { verb: verb, noun: noun } + end + + def draft_action_command + if Feature.disabled?(:draft_quick_action_non_toggle, quick_action_target.project) + quick_action_target.draft? ? 'ready' : 'draft' + else + 'draft' + end + end + def merge_orchestration_service @merge_orchestration_service ||= ::MergeRequests::MergeOrchestrationService.new(project, current_user) end diff --git a/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb new file mode 100644 index 00000000000..e8002656ff5 --- /dev/null +++ b/lib/gitlab/quick_actions/timeline_text_and_date_time_separator.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module QuickActions + class TimelineTextAndDateTimeSeparator + DATETIME_REGEX = %r{(\d{2,4}[\-.]\d{1,2}[\-.]\d{1,2} \d{1,2}:\d{2})}.freeze + MIXED_DELIMITER = %r{([/.])}.freeze + TIME_REGEX = %r{(\d{1,2}:\d{2})}.freeze + + def initialize(timeline_event_arg) + @timeline_event_arg = timeline_event_arg + @timeline_text = get_text + @timeline_date_string = get_raw_date_string + end + + def execute + return if @timeline_event_arg.blank? + return if date_contains_mixed_delimiters? + return [@timeline_text, get_current_date_time] unless date_time_present? + return unless valid_date? + + [@timeline_text, get_actual_date_time] + end + + private + + def get_text + @timeline_event_arg.split('|')[0]&.strip + end + + def get_raw_date_string + @timeline_event_arg.split('|')[1]&.strip + end + + def get_current_date_time + DateTime.current.strftime("%Y-%m-%d %H:%M:00 UTC") + end + + def get_actual_date_time + DateTime.parse(@timeline_date_string) + end + + def date_time_present? + DATETIME_REGEX =~ @timeline_date_string || TIME_REGEX =~ @timeline_date_string + end + + def date_contains_mixed_delimiters? + MIXED_DELIMITER =~ @timeline_date_string + end + + def valid_date? + get_actual_date_time + rescue Date::Error + nil + end + end + end +end diff --git a/lib/gitlab/reactive_cache_set_cache.rb b/lib/gitlab/reactive_cache_set_cache.rb index 7ccbeadfd8a..2de3c07712f 100644 --- a/lib/gitlab/reactive_cache_set_cache.rb +++ b/lib/gitlab/reactive_cache_set_cache.rb @@ -15,8 +15,10 @@ module Gitlab keys = read(key).map { |value| "#{cache_namespace}:#{value}" } keys << cache_key(key) - redis.pipelined do - keys.each_slice(1000) { |subset| redis.unlink(*subset) } + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.pipelined do |pipeline| + keys.each_slice(1000) { |subset| pipeline.unlink(*subset) } + end end end end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb new file mode 100644 index 00000000000..8857b544364 --- /dev/null +++ b/lib/gitlab/redis.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Redis + # List all Gitlab::Redis::Wrapper descendants that are backed by an actual + # separate redis instance here. + # + # This will make sure the connection pool is initialized on application boot in + # config/initializers/7_redis.rb, instrumented, and used in health- & readiness checks. + ALL_CLASSES = [ + Gitlab::Redis::Cache, + Gitlab::Redis::Queues, + Gitlab::Redis::RateLimiting, + Gitlab::Redis::Sessions, + Gitlab::Redis::SharedState, + Gitlab::Redis::TraceChunks + ].freeze + end +end diff --git a/lib/gitlab/redis/cache.rb b/lib/gitlab/redis/cache.rb index 4ab1024d528..043f14630d5 100644 --- a/lib/gitlab/redis/cache.rb +++ b/lib/gitlab/redis/cache.rb @@ -12,7 +12,7 @@ module Gitlab redis: pool, compress: Gitlab::Utils.to_boolean(ENV.fetch('ENABLE_REDIS_CACHE_COMPRESSION', '1')), namespace: CACHE_NAMESPACE, - expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 2.weeks).to_i # Cache should not grow forever + expires_in: ENV.fetch('GITLAB_RAILS_CACHE_DEFAULT_TTL_SECONDS', 8.hours).to_i # Cache should not grow forever } end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index cdd2ac6100e..a7c36786d2d 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -267,7 +267,7 @@ module Gitlab def same_redis_store? strong_memoize(:same_redis_store) do - # <Redis client v4.4.0 for redis:///path_to/redis/redis.socket/5>" + # <Redis client v4.7.1 for unix:///path_to/redis/redis.socket/5>" primary_store.inspect == secondary_store.inspect end end diff --git a/lib/gitlab/repository_hash_cache.rb b/lib/gitlab/repository_hash_cache.rb index 430f3e8d162..1ecdf506208 100644 --- a/lib/gitlab/repository_hash_cache.rb +++ b/lib/gitlab/repository_hash_cache.rb @@ -83,14 +83,14 @@ module Gitlab full_key = cache_key(key) with do |redis| - results = redis.pipelined do + results = redis.pipelined do |pipeline| # Set each hash key to the provided value hash.each do |h_key, h_value| - redis.hset(full_key, h_key, h_value) + pipeline.hset(full_key, h_key, h_value) end # Update the expiry time for this hset - redis.expire(full_key, expires_in) + pipeline.expire(full_key, expires_in) end results.all? diff --git a/lib/gitlab/repository_set_cache.rb b/lib/gitlab/repository_set_cache.rb index 3061fb96190..33c7d96c45b 100644 --- a/lib/gitlab/repository_set_cache.rb +++ b/lib/gitlab/repository_set_cache.rb @@ -21,14 +21,14 @@ module Gitlab full_key = cache_key(key) with do |redis| - redis.multi do - redis.unlink(full_key) + redis.multi do |multi| + multi.unlink(full_key) # Splitting into groups of 1000 prevents us from creating a too-long # Redis command - value.each_slice(1000) { |subset| redis.sadd(full_key, subset) } + value.each_slice(1000) { |subset| multi.sadd(full_key, subset) } - redis.expire(full_key, expires_in) + multi.expire(full_key, expires_in) end end @@ -39,9 +39,9 @@ module Gitlab full_key = cache_key(key) smembers, exists = with do |redis| - redis.multi do - redis.smembers(full_key) - redis.exists(full_key) + redis.multi do |multi| + multi.smembers(full_key) + multi.exists(full_key) end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index a84a6ac2d14..258c904290d 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,6 +6,7 @@ module Gitlab module RequestForgeryProtection + # rubocop:disable Rails/ApplicationController class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true @@ -31,5 +32,6 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end + # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 2450ad88bbb..ec514adafc8 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -151,48 +151,6 @@ module Gitlab model.logger = old_loggers[connection_name] end end - - module Ci - class DailyBuildGroupReportResult - DEFAULT_BRANCH = 'master' - COUNT_OF_DAYS = 5 - - def initialize(project) - @project = project - @last_pipeline = project.last_pipeline - end - - def seed - COUNT_OF_DAYS.times do |count| - date = Time.now.utc - count.day - create_report(date) - end - end - - private - - attr_reader :project, :last_pipeline - - def create_report(date) - last_pipeline.builds.uniq(&:group_name).each do |build| - ::Ci::DailyBuildGroupReportResult.create( - project: project, - last_pipeline: last_pipeline, - date: date, - ref_path: last_pipeline.source_ref_path, - group_name: build.group_name, - data: { - 'coverage' => rand(20..99) - }, - group: project.group, - default_branch: last_pipeline.default_branch? - ) - rescue ActiveRecord::RecordNotUnique - return false - end - end - end - end end end # :nocov: diff --git a/lib/gitlab/seeders/ci/daily_build_group_report_result.rb b/lib/gitlab/seeders/ci/daily_build_group_report_result.rb new file mode 100644 index 00000000000..10ec65f6bf4 --- /dev/null +++ b/lib/gitlab/seeders/ci/daily_build_group_report_result.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + class DailyBuildGroupReportResult + DEFAULT_BRANCH = 'master' + COUNT_OF_DAYS = 5 + + def initialize(project) + @project = project + @last_pipeline = project.last_pipeline + end + + def seed + COUNT_OF_DAYS.times do |count| + date = Time.now.utc - count.day + create_report(date) + end + end + + private + + attr_reader :project, :last_pipeline + + def create_report(date) + last_pipeline.builds.uniq(&:group_name).each do |build| + ::Ci::DailyBuildGroupReportResult.create( + project: project, + last_pipeline: last_pipeline, + date: date, + ref_path: last_pipeline.source_ref_path, + group_name: build.group_name, + data: { + 'coverage' => rand(20..99) + }, + group: project.group, + default_branch: last_pipeline.default_branch? + ) + rescue ActiveRecord::RecordNotUnique + return false + end + end + end + end + end +end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index 896e7e3f65e..23c23393bc8 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -33,10 +33,10 @@ module Gitlab def write(key, value) with do |redis| - redis.pipelined do - redis.sadd(cache_key(key), value) + redis.pipelined do |pipeline| + pipeline.sadd(cache_key(key), value) - redis.expire(cache_key(key), expires_in) + pipeline.expire(cache_key(key), expires_in) end end @@ -57,9 +57,9 @@ module Gitlab full_key = cache_key(key) with do |redis| - redis.multi do - redis.sismember(full_key, value) - redis.exists(full_key) + redis.multi do |multi| + multi.sismember(full_key, value) + multi.exists(full_key) end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index d26e1a34a9f..b167afe589a 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -70,7 +70,9 @@ module Gitlab link_path = File.join(shell_path, '.gitlab_shell_secret') if File.exist?(shell_path) && !File.exist?(link_path) - FileUtils.symlink(secret_file, link_path) + # It could happen that link_path is a broken symbolic link. + # In that case !File.exist?(link_path) is true, but we still want to overwrite the (broken) symbolic link. + FileUtils.ln_sf(secret_file, link_path) end end end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index ca92fed9c40..24e2eca420e 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -41,11 +41,11 @@ module Gitlab def init_metrics { - sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), + sidekiq_current_rss: ::Gitlab::Metrics.gauge(:sidekiq_current_rss, 'Current RSS of Sidekiq Worker'), sidekiq_memory_killer_soft_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_soft_limit_rss, 'Current soft_limit_rss of Sidekiq Worker'), sidekiq_memory_killer_hard_limit_rss: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_hard_limit_rss, 'Current hard_limit_rss of Sidekiq Worker'), - sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'), - sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached') + sidekiq_memory_killer_phase: ::Gitlab::Metrics.gauge(:sidekiq_memory_killer_phase, 'Current phase of Sidekiq Worker'), + sidekiq_memory_killer_running_jobs: ::Gitlab::Metrics.counter(:sidekiq_memory_killer_running_jobs_total, 'Current running jobs when limit was reached') } end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 7533770e254..ab126ea4749 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -112,10 +112,12 @@ module Gitlab end def delete! - with_redis do |redis| - redis.multi do |multi| - multi.del(idempotency_key, deduplicated_flag_key) - delete_wal_locations!(multi) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + with_redis do |redis| + redis.multi do |multi| + multi.del(idempotency_key, deduplicated_flag_key) + delete_wal_locations!(multi) + end end end end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 180cdad916b..3dd5355d3a3 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -22,21 +22,21 @@ module Gitlab def metrics { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), + sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_completion_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS), + sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), + sidekiq_jobs_queue_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS), sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), - sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), - sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), - sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), - sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), - sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), - sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all), - sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all) + sidekiq_jobs_failed_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'), + sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), + sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), + sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), + sidekiq_elasticsearch_requests_total: ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'), + sidekiq_running_jobs: ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all), + sidekiq_concurrency: ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all), + sidekiq_mem_total_bytes: ::Gitlab::Metrics.gauge(:sidekiq_mem_total_bytes, 'Number of bytes allocated for both objects consuming an object slot and objects that required a malloc', {}, :all) } end diff --git a/lib/gitlab/sidekiq_versioning.rb b/lib/gitlab/sidekiq_versioning.rb index 80c0b7650f3..28c9714f82f 100644 --- a/lib/gitlab/sidekiq_versioning.rb +++ b/lib/gitlab/sidekiq_versioning.rb @@ -10,11 +10,7 @@ module Gitlab if queues.any? Sidekiq.redis do |conn| - conn.pipelined do - queues.each do |queue| - conn.sadd('queues', queue) - end - end + conn.sadd('queues', queues) end end rescue ::Redis::BaseError, SocketError, Errno::ENOENT, Errno::EADDRNOTAVAIL, Errno::EAFNOSUPPORT, Errno::ECONNRESET, Errno::ECONNREFUSED diff --git a/lib/gitlab/slash_commands/presenters/base.rb b/lib/gitlab/slash_commands/presenters/base.rb index d28b5fb509a..55497c5e365 100644 --- a/lib/gitlab/slash_commands/presenters/base.rb +++ b/lib/gitlab/slash_commands/presenters/base.rb @@ -87,16 +87,16 @@ module Gitlab { attachments: [ { - title: "#{issue.title} · #{issue.to_reference}", - title_link: resource_url, - author_name: author.name, - author_icon: author.avatar_url(only_path: false), - fallback: fallback_message, - pretext: custom_pretext, - text: text, - color: color(resource), - fields: fields, - mrkdwn_in: fields_with_markdown + title: "#{issue.title} · #{issue.to_reference}", + title_link: resource_url, + author_name: author.name, + author_icon: author.avatar_url(only_path: false), + fallback: fallback_message, + pretext: custom_pretext, + text: text, + color: color(resource), + fields: fields, + mrkdwn_in: fields_with_markdown } ] } diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 40b01552244..0b9f3baa4de 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -33,33 +33,50 @@ module Gitlab @endpoint_url = @endpoint_url.sub(URL_SCHEME_REGEX, '') end - def issue_spam?(spam_issue:, user:, context: {}) - issue = build_issue_protobuf(issue: spam_issue, user: user, context: context) + def spam?(spammable:, user:, context: {}, extra_features: {}) + metadata = { 'authorization' => Gitlab::CurrentSettings.spam_check_api_key || '' } + protobuf_args = { spammable: spammable, user: user, context: context, extra_features: extra_features } + + pb, grpc_method = build_protobuf(**protobuf_args) + response = grpc_method.call(pb, metadata: metadata) - response = grpc_client.check_for_spam_issue(issue, - metadata: { 'authorization' => - Gitlab::CurrentSettings.spam_check_api_key }) verdict = convert_verdict_to_gitlab_constant(response.verdict) [verdict, response.extra_attributes.to_h, response.error] end private + def get_spammable_mappings(spammable) + case spammable + when Issue + [::Spamcheck::Issue, grpc_client.method(:check_for_spam_issue)] + when Snippet + [::Spamcheck::Snippet, grpc_client.method(:check_for_spam_snippet)] + else + raise ArgumentError, "Not a spammable type: #{spammable.class.name}" + end + end + def convert_verdict_to_gitlab_constant(verdict) VERDICT_MAPPING.fetch(::Spamcheck::SpamVerdict::Verdict.resolve(verdict), verdict) end - def build_issue_protobuf(issue:, user:, context:) - issue_pb = ::Spamcheck::Issue.new - issue_pb.title = issue.spam_title || '' - issue_pb.description = issue.spam_description || '' - issue_pb.created_at = convert_to_pb_timestamp(issue.created_at) if issue.created_at - issue_pb.updated_at = convert_to_pb_timestamp(issue.updated_at) if issue.updated_at - issue_pb.user_in_project = user.authorized_project?(issue.project) - issue_pb.project = build_project_protobuf(issue) - issue_pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) - issue_pb.user = build_user_protobuf(user) - issue_pb + def build_protobuf(spammable:, user:, context:, extra_features:) + protobuf_class, grpc_method = get_spammable_mappings(spammable) + pb = protobuf_class.new(**extra_features) + pb.title = spammable.spam_title || '' + pb.description = spammable.spam_description || '' + pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at + pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at + pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) + pb.user = build_user_protobuf(user) + + unless spammable.project.nil? + pb.user_in_project = user.authorized_project?(spammable.project) + pb.project = build_project_protobuf(spammable) + end + + [pb, grpc_method] end def build_user_protobuf(user) diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 7ef1be6ff44..7494f0584d0 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -22,6 +22,10 @@ module Gitlab "payment_method_validation" end + def self.registration_validation_form_id + "cc_registration_validation" + end + def self.registration_validation_form_url "#{self.subscriptions_url}/payment_forms/cc_registration_validation" end @@ -90,3 +94,4 @@ Gitlab::SubscriptionPortal::PAYMENT_FORM_URL = Gitlab::SubscriptionPortal.paymen Gitlab::SubscriptionPortal::PAYMENT_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.payment_validation_form_id.freeze Gitlab::SubscriptionPortal::RENEWAL_SERVICE_EMAIL = Gitlab::SubscriptionPortal.renewal_service_email.freeze Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_URL = Gitlab::SubscriptionPortal.registration_validation_form_url.freeze +Gitlab::SubscriptionPortal::REGISTRATION_VALIDATION_FORM_ID = Gitlab::SubscriptionPortal.registration_validation_form_id.freeze diff --git a/lib/gitlab/template/gitignore_template.rb b/lib/gitlab/template/gitignore_template.rb index 72a1b7460c2..d8e0ec82410 100644 --- a/lib/gitlab/template/gitignore_template.rb +++ b/lib/gitlab/template/gitignore_template.rb @@ -11,7 +11,7 @@ module Gitlab def categories { "Languages" => '', - "Global" => 'Global' + "Global" => 'Global' } end diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index 3b46b4c5498..45f836f10d3 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -10,6 +10,8 @@ module Gitlab def event(category, action, label: nil, property: nil, value: nil, context: [], project: nil, user: nil, namespace: nil, **extra) # rubocop:disable Metrics/ParameterLists contexts = [Tracking::StandardContext.new(project: project, user: user, namespace: namespace, **extra).to_context, *context] + action = action.to_s + tracker.event(category, action, label: label, property: property, value: value, context: contexts) rescue StandardError => error Gitlab::ErrorTracking.track_and_raise_for_dev_exception(error, snowplow_category: category, snowplow_action: action) diff --git a/lib/gitlab/tree_summary.rb b/lib/gitlab/tree_summary.rb index 72df8b423df..ba3176ca6e7 100644 --- a/lib/gitlab/tree_summary.rb +++ b/lib/gitlab/tree_summary.rb @@ -6,7 +6,7 @@ module Gitlab include ::MarkupHelper CACHE_EXPIRE_IN = 1.hour - MAX_OFFSET = 2**31 + MAX_OFFSET = 2**31 - 1 attr_reader :commit, :project, :path, :offset, :limit, :user, :resolved_commits @@ -35,6 +35,8 @@ module Gitlab # - commit_path: URI of the commit in the web interface # - commit_title_html: Rendered commit title def summarize + return [] if offset < 0 + commits_hsh = fetch_last_cached_commits_list prerender_commit_full_titles!(commits_hsh.values) diff --git a/lib/gitlab/uploads/migration_helper.rb b/lib/gitlab/uploads/migration_helper.rb index deab2cd43a6..712512d0e02 100644 --- a/lib/gitlab/uploads/migration_helper.rb +++ b/lib/gitlab/uploads/migration_helper.rb @@ -5,27 +5,10 @@ module Gitlab class MigrationHelper attr_reader :logger - CATEGORIES = [%w(AvatarUploader Project :avatar), - %w(AvatarUploader Group :avatar), - %w(AvatarUploader User :avatar), - %w(AttachmentUploader Note :attachment), - %w(AttachmentUploader Appearance :logo), - %w(AttachmentUploader Appearance :header_logo), - %w(FaviconUploader Appearance :favicon), - %w(FileUploader Project), - %w(PersonalFileUploader Snippet), - %w(NamespaceFileUploader Snippet), - %w(DesignManagement::DesignV432x230Uploader DesignManagement::Action :image_v432x230), - %w(FileUploader MergeRequest)].freeze - def initialize(args, logger) prepare_variables(args, logger) end - def self.categories - CATEGORIES - end - def migrate_to_remote_storage @to_store = ObjectStorage::Store::REMOTE @@ -45,17 +28,14 @@ module Gitlab end def prepare_variables(args, logger) - @mounted_as = args.mounted_as&.gsub(':', '')&.to_sym - @uploader_class = args.uploader_class.constantize - @model_class = args.model_class.constantize + @mounted_as = args.mounted_as&.gsub(':', '') + @uploader_class = args.uploader_class + @model_class = args.model_class&.constantize @logger = logger end def enqueue_batch(batch, index) - job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, - @model_class, - @mounted_as, - @to_store) + job = ObjectStorage::MigrateUploadsWorker.enqueue!(batch, @to_store) logger.info(message: "[Uploads migration] Enqueued upload migration job", index: index, job_id: job) rescue ObjectStorage::MigrateUploadsWorker::SanityCheckError => e # continue for the next batch @@ -66,10 +46,12 @@ module Gitlab def uploads(store_type = [nil, ObjectStorage::Store::LOCAL]) Upload.class_eval { include EachBatch } unless Upload < EachBatch - Upload - .where(store: store_type, - uploader: @uploader_class.to_s, - model_type: @model_class.base_class.sti_name) + uploads = Upload.where(store: store_type) + uploads = uploads.where(uploader: @uploader_class) if @uploader_class.present? + uploads = uploads.where(model_type: @model_class.base_class.sti_name) if @model_class.present? + uploads = uploads.where(mount_point: @mounted_as) if @mounted_as.present? + + uploads end # rubocop:enable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb index c0d53b1b21a..67dc1455b23 100644 --- a/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/count_bulk_imports_entities_metric.rb @@ -20,15 +20,20 @@ module Gitlab private def relation - return super.where(source_type: source_type) if source_type.present? # rubocop: disable CodeReuse/ActiveRecord - - super + scope = super + scope = scope.where(source_type: source_type) if source_type.present? + scope = scope.where(status: status) if status.present? + scope end def source_type options[:source_type].to_s end + def status + options[:status] + end + def allowed_source_types BulkImports::Entity.source_types.keys.map(&:to_s) end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb new file mode 100644 index 00000000000..1de93ce6dfa --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_user_auth_metric.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountUserAuthMetric < DatabaseMetric + operation :distinct_count, column: :user_id + + relation do + AuthenticationEvent.success + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb index a25bad2436b..26d963e2407 100644 --- a/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/redis_metric.rb @@ -11,37 +11,49 @@ module Gitlab # instrumentation_class: RedisMetric # options: # event: pushes - # counter_class: SourceCodeCounter + # prefix: source_code # class RedisMetric < BaseMetric + include Gitlab::UsageDataCounters::RedisCounter + + USAGE_PREFIX = "USAGE_" + def initialize(time_frame:, options: {}) super raise ArgumentError, "'event' option is required" unless metric_event.present? - raise ArgumentError, "'counter class' option is required" unless counter_class.present? + raise ArgumentError, "'prefix' option is required" unless prefix.present? end def metric_event options[:event] end - def counter_class_name - options[:counter_class] + def prefix + options[:prefix] end - def counter_class - "Gitlab::UsageDataCounters::#{counter_class_name}".constantize + def include_usage_prefix? + options.fetch(:include_usage_prefix, true) end def value redis_usage_data do - counter_class.read(metric_event) + total_count(redis_key) end end def suggested_name Gitlab::Usage::Metrics::NameSuggestion.for(:redis) end + + private + + def redis_key + key = "#{prefix}_#{metric_event}".upcase + key.prepend(USAGE_PREFIX) if include_usage_prefix? + key + end end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 6f36a09fe48..e2232dc5e2a 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -137,7 +137,7 @@ module Gitlab projects_with_error_tracking_enabled: count(::ErrorTracking::ProjectErrorTrackingSetting.where(enabled: true)), projects_with_alerts_created: distinct_count(::AlertManagement::Alert, :project_id), projects_with_enabled_alert_integrations: distinct_count(::AlertManagement::HttpIntegration.active, :project_id), - projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.terraform_reports, :project_id), + projects_with_terraform_reports: distinct_count(::Ci::JobArtifact.of_report_type(:terraform), :project_id), projects_with_terraform_states: distinct_count(::Terraform::State, :project_id), protected_branches: count(ProtectedBranch), protected_branches_except_default: count(ProtectedBranch.where.not(name: ['main', 'master', Gitlab::CurrentSettings.default_branch_name])), @@ -146,7 +146,7 @@ module Gitlab personal_snippets: count(PersonalSnippet), project_snippets: count(ProjectSnippet), suggestions: count(Suggestion), - terraform_reports: count(::Ci::JobArtifact.terraform_reports), + terraform_reports: count(::Ci::JobArtifact.of_report_type(:terraform)), terraform_states: count(::Terraform::State), todos: count(Todo), uploads: count(Upload), @@ -268,7 +268,7 @@ module Gitlab # @return [Array<#totals>] An array of objects that respond to `#totals` def usage_data_counters - Gitlab::UsageDataCounters.counters + Gitlab::UsageDataCounters.unmigrated_counters end def components_usage_data diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index 224897ed758..eae1c593a8f 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -3,29 +3,38 @@ module Gitlab module UsageDataCounters COUNTERS = [ - PackageEventCounter, WikiPageCounter, - WebIdeCounter, NoteCounter, SnippetCounter, SearchCounter, CycleAnalyticsCounter, ProductivityAnalyticsCounter, SourceCodeCounter, + KubernetesAgentCounter, + MergeRequestWidgetExtensionCounter + ].freeze + + COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ + PackageEventCounter, MergeRequestCounter, DesignsCounter, - KubernetesAgentCounter, DiffsCounter, ServiceUsageDataCounter, - MergeRequestWidgetExtensionCounter + WebIdeCounter ].freeze UsageDataCounterError = Class.new(StandardError) UnknownEvent = Class.new(UsageDataCounterError) class << self + def unmigrated_counters + # we are using the #counters method instead of the COUNTERS const + # to make sure it's working correctly for `ee` version of UsageDataCounters + counters - self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES + end + def counters - self::COUNTERS + self::COUNTERS + self::COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES end def count(event_name) diff --git a/lib/gitlab/usage_data_counters/base_counter.rb b/lib/gitlab/usage_data_counters/base_counter.rb index 4ab310a2519..5d2ab5eaf74 100644 --- a/lib/gitlab/usage_data_counters/base_counter.rb +++ b/lib/gitlab/usage_data_counters/base_counter.rb @@ -10,7 +10,9 @@ module Gitlab::UsageDataCounters def redis_key(event) require_known_event(event) - "USAGE_#{prefix}_#{event}".upcase + usage_prefix = Gitlab::Usage::Metrics::Instrumentations::RedisMetric::USAGE_PREFIX + + "#{usage_prefix}#{prefix}_#{event}".upcase end def count(event) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index a5db8ba4dcc..f0cb9bcbe94 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -20,11 +20,7 @@ module Gitlab CATEGORIES_FOR_TOTALS = %w[ analytics - code_review compliance - deploy_token_packages - ecosystem - epic_boards_usage epics_usage error_tracking ide_edit @@ -32,11 +28,13 @@ module Gitlab issues_edit pipeline_authoring quickactions - user_packages ].freeze CATEGORIES_COLLECTED_FROM_METRICS_DEFINITIONS = %w[ ci_users + deploy_token_packages + code_review + ecosystem error_tracking ide_edit importer @@ -49,6 +47,7 @@ module Gitlab source_code terraform testing + user_packages work_items ].freeze diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 316d9bb3dc1..dda72f7fa3b 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -36,95 +36,118 @@ module Gitlab ISSUE_COMMENT_REMOVED = 'g_project_management_issue_comment_removed' class << self - def track_issue_created_action(author:) + def track_issue_created_action(author:, project:) + track_snowplow_action(ISSUE_CREATED, author, project) track_unique_action(ISSUE_CREATED, author) end - def track_issue_title_changed_action(author:) + def track_issue_title_changed_action(author:, project:) + track_snowplow_action(ISSUE_TITLE_CHANGED, author, project) track_unique_action(ISSUE_TITLE_CHANGED, author) end - def track_issue_description_changed_action(author:) + def track_issue_description_changed_action(author:, project:) + track_snowplow_action(ISSUE_DESCRIPTION_CHANGED, author, project) track_unique_action(ISSUE_DESCRIPTION_CHANGED, author) end - def track_issue_assignee_changed_action(author:) + def track_issue_assignee_changed_action(author:, project:) + track_snowplow_action(ISSUE_ASSIGNEE_CHANGED, author, project) track_unique_action(ISSUE_ASSIGNEE_CHANGED, author) end - def track_issue_made_confidential_action(author:) + def track_issue_made_confidential_action(author:, project:) + track_snowplow_action(ISSUE_MADE_CONFIDENTIAL, author, project) track_unique_action(ISSUE_MADE_CONFIDENTIAL, author) end - def track_issue_made_visible_action(author:) + def track_issue_made_visible_action(author:, project:) + track_snowplow_action(ISSUE_MADE_VISIBLE, author, project) track_unique_action(ISSUE_MADE_VISIBLE, author) end - def track_issue_closed_action(author:) + def track_issue_closed_action(author:, project:) + track_snowplow_action(ISSUE_CLOSED, author, project) track_unique_action(ISSUE_CLOSED, author) end - def track_issue_reopened_action(author:) + def track_issue_reopened_action(author:, project:) + track_snowplow_action(ISSUE_REOPENED, author, project) track_unique_action(ISSUE_REOPENED, author) end - def track_issue_label_changed_action(author:) + def track_issue_label_changed_action(author:, project:) + track_snowplow_action(ISSUE_LABEL_CHANGED, author, project) track_unique_action(ISSUE_LABEL_CHANGED, author) end - def track_issue_milestone_changed_action(author:) + def track_issue_milestone_changed_action(author:, project:) + track_snowplow_action(ISSUE_MILESTONE_CHANGED, author, project) track_unique_action(ISSUE_MILESTONE_CHANGED, author) end - def track_issue_cross_referenced_action(author:) + def track_issue_cross_referenced_action(author:, project:) + track_snowplow_action(ISSUE_CROSS_REFERENCED, author, project) track_unique_action(ISSUE_CROSS_REFERENCED, author) end - def track_issue_moved_action(author:) + def track_issue_moved_action(author:, project:) + track_snowplow_action(ISSUE_MOVED, author, project) track_unique_action(ISSUE_MOVED, author) end - def track_issue_related_action(author:) + def track_issue_related_action(author:, project:) + track_snowplow_action(ISSUE_RELATED, author, project) track_unique_action(ISSUE_RELATED, author) end - def track_issue_unrelated_action(author:) + def track_issue_unrelated_action(author:, project:) + track_snowplow_action(ISSUE_UNRELATED, author, project) track_unique_action(ISSUE_UNRELATED, author) end - def track_issue_marked_as_duplicate_action(author:) + def track_issue_marked_as_duplicate_action(author:, project:) + track_snowplow_action(ISSUE_MARKED_AS_DUPLICATE, author, project) track_unique_action(ISSUE_MARKED_AS_DUPLICATE, author) end - def track_issue_locked_action(author:) + def track_issue_locked_action(author:, project:) + track_snowplow_action(ISSUE_LOCKED, author, project) track_unique_action(ISSUE_LOCKED, author) end - def track_issue_unlocked_action(author:) + def track_issue_unlocked_action(author:, project:) + track_snowplow_action(ISSUE_UNLOCKED, author, project) track_unique_action(ISSUE_UNLOCKED, author) end - def track_issue_designs_added_action(author:) + def track_issue_designs_added_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_ADDED, author, project) track_unique_action(ISSUE_DESIGNS_ADDED, author) end - def track_issue_designs_modified_action(author:) + def track_issue_designs_modified_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_MODIFIED, author, project) track_unique_action(ISSUE_DESIGNS_MODIFIED, author) end - def track_issue_designs_removed_action(author:) + def track_issue_designs_removed_action(author:, project:) + track_snowplow_action(ISSUE_DESIGNS_REMOVED, author, project) track_unique_action(ISSUE_DESIGNS_REMOVED, author) end - def track_issue_due_date_changed_action(author:) + def track_issue_due_date_changed_action(author:, project:) + track_snowplow_action(ISSUE_DUE_DATE_CHANGED, author, project) track_unique_action(ISSUE_DUE_DATE_CHANGED, author) end - def track_issue_time_estimate_changed_action(author:) + def track_issue_time_estimate_changed_action(author:, project:) + track_snowplow_action(ISSUE_TIME_ESTIMATE_CHANGED, author, project) track_unique_action(ISSUE_TIME_ESTIMATE_CHANGED, author) end - def track_issue_time_spent_changed_action(author:) + def track_issue_time_spent_changed_action(author:, project:) + track_snowplow_action(ISSUE_TIME_SPENT_CHANGED, author, project) track_unique_action(ISSUE_TIME_SPENT_CHANGED, author) end diff --git a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml index a8f1bab1f20..10e36a75a3a 100644 --- a/lib/gitlab/usage_data_counters/known_events/ci_templates.yml +++ b/lib/gitlab/usage_data_counters/known_events/ci_templates.yml @@ -139,6 +139,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_security_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_security_api_fuzzing category: ci_templates redis_slot: ci_templates @@ -231,6 +235,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_katalon + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_mono category: ci_templates redis_slot: ci_templates @@ -319,6 +327,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_license_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_deploy category: ci_templates redis_slot: ci_templates @@ -331,6 +343,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_jobs_dependency_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_jobs_test category: ci_templates redis_slot: ci_templates @@ -523,6 +539,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_license_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_deploy category: ci_templates redis_slot: ci_templates @@ -535,6 +555,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_jobs_dependency_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_jobs_test category: ci_templates redis_slot: ci_templates @@ -635,6 +659,10 @@ category: ci_templates redis_slot: ci_templates aggregation: weekly +- name: p_ci_templates_implicit_security_container_scanning_latest + category: ci_templates + redis_slot: ci_templates + aggregation: weekly - name: p_ci_templates_implicit_security_api_fuzzing category: ci_templates redis_slot: ci_templates diff --git a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml index c21b99ba834..0bd809f8aa5 100644 --- a/lib/gitlab/usage_data_counters/known_events/code_review_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/code_review_events.yml @@ -1,9 +1,29 @@ --- -- name: i_code_review_mr_diffs +- name: i_code_review_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_create_note_in_ipynb_diff_commit + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_mr + redis_slot: code_review + category: code_review + aggregation: weekly +- name: i_code_review_user_create_note_in_ipynb_diff_commit redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_mr_with_invalid_approvers +- name: i_code_review_mr_diffs redis_slot: code_review category: code_review aggregation: weekly @@ -135,12 +155,10 @@ redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_jetbrains_api_request - name: i_code_review_user_gitlab_cli_api_request redis_slot: code_review category: code_review aggregation: weekly - feature_flag: usage_data_i_code_review_user_gitlab_cli_api_request - name: i_code_review_user_create_mr_from_issue redis_slot: code_review category: code_review @@ -177,30 +195,6 @@ redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_mr - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_user_create_note_in_ipynb_diff_commit - redis_slot: code_review - category: code_review - aggregation: weekly # Diff settings events - name: i_code_review_click_diff_view_setting redis_slot: code_review @@ -400,53 +394,36 @@ redis_slot: code_review category: code_review aggregation: weekly -## Metrics -- name: i_code_review_merge_request_widget_metrics_view - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_full_report_clicked - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_success - redis_slot: code_review - category: code_review - aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_warning +- name: i_code_review_submit_review_approve redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_metrics_expand_failed +- name: i_code_review_submit_review_comment redis_slot: code_review category: code_review aggregation: weekly -## Status Checks -- name: i_code_review_merge_request_widget_status_checks_view +## License Compliance +- name: i_code_review_merge_request_widget_license_compliance_view redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_full_report_clicked +- name: i_code_review_merge_request_widget_license_compliance_full_report_clicked redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand +- name: i_code_review_merge_request_widget_license_compliance_expand redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_success +- name: i_code_review_merge_request_widget_license_compliance_expand_success redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_warning +- name: i_code_review_merge_request_widget_license_compliance_expand_warning redis_slot: code_review category: code_review aggregation: weekly -- name: i_code_review_merge_request_widget_status_checks_expand_failed +- name: i_code_review_merge_request_widget_license_compliance_expand_failed redis_slot: code_review category: code_review aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index 6c4754ae19f..29b231f88f8 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -146,6 +146,11 @@ category: testing redis_slot: testing aggregation: weekly +- name: i_testing_test_report_uploaded + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_ci_i_testing_test_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit @@ -332,11 +337,6 @@ redis_slot: testing category: testing aggregation: weekly -# Container Security - Network Policies -- name: clusters_using_network_policies_ui - redis_slot: network_policies - category: network_policies - aggregation: weekly # Geo group - name: g_geo_proxied_requests category: geo @@ -352,3 +352,8 @@ category: manage aggregation: weekly expiry: 42 +# Environments page +- name: users_visiting_environments_pages + category: environments + redis_slot: users + aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml index f594c6a1b7c..7f7c9166086 100644 --- a/lib/gitlab/usage_data_counters/known_events/ecosystem.yml +++ b/lib/gitlab/usage_data_counters/known_events/ecosystem.yml @@ -8,14 +8,6 @@ category: ecosystem redis_slot: ecosystem aggregation: weekly -- name: i_ecosystem_jira_service_list_issues - category: ecosystem - redis_slot: ecosystem - aggregation: weekly -- name: i_ecosystem_jira_service_create_issue - category: ecosystem - redis_slot: ecosystem - aggregation: weekly - name: i_ecosystem_slack_service_issue_notification category: ecosystem redis_slot: ecosystem diff --git a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml b/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml deleted file mode 100644 index 3879c561cc4..00000000000 --- a/lib/gitlab/usage_data_counters/known_events/epic_board_events.yml +++ /dev/null @@ -1,19 +0,0 @@ -# Epic board events -# -# We are using the same slot of issue events 'project_management' for -# epic events to allow data aggregation. -# More information in: https://gitlab.com/gitlab-org/gitlab/-/issues/322405 -- name: g_project_management_users_creating_epic_boards - category: epic_boards_usage - redis_slot: project_management - aggregation: daily - -- name: g_project_management_users_viewing_epic_boards - category: epic_boards_usage - redis_slot: project_management - aggregation: daily - -- name: g_project_management_users_updating_epic_board_names - category: epic_boards_usage - redis_slot: project_management - aggregation: daily diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index e1de74a3d07..966e6c584c7 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -2,4 +2,3 @@ category: kubernetes_agent redis_slot: agent aggregation: weekly - feature_flag: track_agent_users_using_ci_tunnel diff --git a/lib/gitlab/usage_data_counters/known_events/quickactions.yml b/lib/gitlab/usage_data_counters/known_events/quickactions.yml index f980503b4bf..58a0c0695af 100644 --- a/lib/gitlab/usage_data_counters/known_events/quickactions.yml +++ b/lib/gitlab/usage_data_counters/known_events/quickactions.yml @@ -127,6 +127,10 @@ category: quickactions redis_slot: quickactions aggregation: weekly +- name: i_quickactions_timeline + category: quickactions + redis_slot: quickactions + aggregation: weekly - name: i_quickactions_page category: quickactions redis_slot: quickactions @@ -303,11 +307,3 @@ category: quickactions redis_slot: quickactions aggregation: weekly -- name: i_quickactions_attention - category: quickactions - redis_slot: quickactions - aggregation: weekly -- name: i_quickactions_remove_attention - category: quickactions - redis_slot: quickactions - aggregation: weekly diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index fbb03a31a6f..93137b762ec 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -49,6 +49,8 @@ module Gitlab MR_LOAD_CONFLICT_UI_ACTION = 'i_code_review_user_load_conflict_ui' MR_RESOLVE_CONFLICT_ACTION = 'i_code_review_user_resolve_conflict' MR_RESOLVE_THREAD_IN_ISSUE_ACTION = 'i_code_review_user_resolve_thread_in_issue' + MR_SUBMIT_REVIEW_APPROVE = 'i_code_review_submit_review_approve' + MR_SUBMIT_REVIEW_COMMENT = 'i_code_review_submit_review_comment' class << self def track_mr_diffs_action(merge_request:) @@ -230,6 +232,14 @@ module Gitlab track_unique_action_by_user(MR_RESOLVE_THREAD_IN_ISSUE_ACTION, user) end + def track_submit_review_approve(user:) + track_unique_action_by_user(MR_SUBMIT_REVIEW_APPROVE, user) + end + + def track_submit_review_comment(user:) + track_unique_action_by_user(MR_SUBMIT_REVIEW_COMMENT, user) + end + private def track_unique_action_by_merge_request(action, merge_request) diff --git a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb index dafc36ab7ce..f88bbc41c70 100644 --- a/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_widget_extension_counter.rb @@ -5,7 +5,7 @@ module Gitlab class MergeRequestWidgetExtensionCounter < BaseCounter KNOWN_EVENTS = %w[view full_report_clicked expand expand_success expand_warning expand_failed].freeze PREFIX = 'i_code_review_merge_request_widget' - WIDGETS = %w[accessibility code_quality status_checks terraform test_summary metrics].freeze + WIDGETS = %w[accessibility code_quality license_compliance status_checks terraform test_summary metrics].freeze class << self private diff --git a/lib/gitlab/utils/deep_size.rb b/lib/gitlab/utils/deep_size.rb index e185786e638..20f2d699e2b 100644 --- a/lib/gitlab/utils/deep_size.rb +++ b/lib/gitlab/utils/deep_size.rb @@ -25,10 +25,6 @@ module Gitlab !too_big? && !too_deep? end - def self.human_default_max_size - ActiveSupport::NumberHelper.number_to_human_size(DEFAULT_MAX_SIZE) - end - private def evaluate diff --git a/lib/gitlab/utils/execution_tracker.rb b/lib/gitlab/utils/execution_tracker.rb new file mode 100644 index 00000000000..6d48658853c --- /dev/null +++ b/lib/gitlab/utils/execution_tracker.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Gitlab + module Utils + class ExecutionTracker + MAX_RUNTIME = 30.seconds + + ExecutionTimeOutError = Class.new(StandardError) + + delegate :monotonic_time, to: :'Gitlab::Metrics::System' + + def initialize + @start_time = monotonic_time + end + + def over_limit? + monotonic_time - start_time >= MAX_RUNTIME + end + + private + + attr_reader :start_time + end + end +end diff --git a/lib/gitlab/view/presenter/base.rb b/lib/gitlab/view/presenter/base.rb index a2d217fb42f..2a57ca9ae02 100644 --- a/lib/gitlab/view/presenter/base.rb +++ b/lib/gitlab/view/presenter/base.rb @@ -46,6 +46,13 @@ module Gitlab url_builder.build(__subject__, only_path: true) end + def path_with_line_numbers(path, start_line, end_line) + path.tap do |complete_path| + complete_path << "#L#{start_line}" + complete_path << "-#{end_line}" if end_line && end_line != start_line + end + end + class_methods do def presenter? true diff --git a/lib/gitlab/visibility_level.rb b/lib/gitlab/visibility_level.rb index 049e3befe64..7360585df43 100644 --- a/lib/gitlab/visibility_level.rb +++ b/lib/gitlab/visibility_level.rb @@ -47,17 +47,17 @@ module Gitlab def options { - s_('VisibilityLevel|Private') => PRIVATE, + s_('VisibilityLevel|Private') => PRIVATE, s_('VisibilityLevel|Internal') => INTERNAL, - s_('VisibilityLevel|Public') => PUBLIC + s_('VisibilityLevel|Public') => PUBLIC } end def string_options { - 'private' => PRIVATE, + 'private' => PRIVATE, 'internal' => INTERNAL, - 'public' => PUBLIC + 'public' => PUBLIC } end diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb index 1b5350d4a4e..031d9ec6ec4 100644 --- a/lib/gitlab/web_hooks/recursion_detection.rb +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -40,9 +40,9 @@ module Gitlab cache_key = cache_key_for_hook(hook) ::Gitlab::Redis::SharedState.with do |redis| - redis.multi do - redis.sadd(cache_key, hook.id) - redis.expire(cache_key, TOUCH_CACHE_TTL) + redis.multi do |multi| + multi.sadd(cache_key, hook.id) + multi.expire(cache_key, TOUCH_CACHE_TTL) end end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index e81670ce89a..906439d5e71 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -12,7 +12,7 @@ module Gitlab VERSION_FILE = 'GITLAB_WORKHORSE_VERSION' INTERNAL_API_CONTENT_TYPE = 'application/vnd.gitlab-workhorse+json' INTERNAL_API_REQUEST_HEADER = 'Gitlab-Workhorse-Api-Request' - NOTIFICATION_CHANNEL = 'workhorse:notifications' + NOTIFICATION_PREFIX = 'workhorse:notifications:' ALLOWED_GIT_HTTP_ACTIONS = %w[git_receive_pack git_upload_pack info_refs].freeze DETECT_HEADER = 'Gitlab-Workhorse-Detect-Content-Type' ARCHIVE_FORMATS = %w(zip tar.gz tar.bz2 tar).freeze @@ -217,7 +217,8 @@ module Gitlab Gitlab::Redis::SharedState.with do |redis| result = redis.set(key, value, ex: expire, nx: !overwrite) if result - redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") + redis.publish(NOTIFICATION_PREFIX + key, value) + value else redis.get(key) |