diff options
Diffstat (limited to 'lib/gitlab')
231 files changed, 9696 insertions, 2953 deletions
diff --git a/lib/gitlab/access/branch_protection.rb b/lib/gitlab/access/branch_protection.rb index 6ac8de407b0..81f02c004af 100644 --- a/lib/gitlab/access/branch_protection.rb +++ b/lib/gitlab/access/branch_protection.rb @@ -45,6 +45,63 @@ module Gitlab def fully_protected? level == PROTECTION_FULL end + + def to_hash + # translate the original integer values into a json payload + # that matches the protected branches API: + # https://docs.gitlab.com/ee/api/protected_branches.html#update-a-protected-branch + case level + when PROTECTION_NONE + self.class.protection_none + when PROTECTION_DEV_CAN_PUSH + self.class.protection_partial + when PROTECTION_FULL + self.class.protected_fully + when PROTECTION_DEV_CAN_MERGE + self.class.protected_against_developer_pushes + when PROTECTION_DEV_CAN_INITIAL_PUSH + self.class.protected_after_initial_push + end + end + + class << self + def protection_none + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protection_partial + protection_none.merge(allow_force_push: false) + end + + def protected_fully + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allow_force_push: false + } + end + + def protected_against_developer_pushes + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true + } + end + + def protected_after_initial_push + { + allowed_to_push: [{ 'access_level' => Gitlab::Access::MAINTAINER }], + allowed_to_merge: [{ 'access_level' => Gitlab::Access::DEVELOPER }], + allow_force_push: true, + developer_can_initial_push: true + } + end + end end end end diff --git a/lib/gitlab/alert_management/payload/base.rb b/lib/gitlab/alert_management/payload/base.rb index 5b136431ce7..3840a560c57 100644 --- a/lib/gitlab/alert_management/payload/base.rb +++ b/lib/gitlab/alert_management/payload/base.rb @@ -33,7 +33,6 @@ module Gitlab :has_required_attributes?, :hosts, :metric_id, - :metrics_dashboard_url, :monitoring_tool, :resolved?, :runbook, diff --git a/lib/gitlab/alert_management/payload/managed_prometheus.rb b/lib/gitlab/alert_management/payload/managed_prometheus.rb index 2236e60a0c6..4ed21108d3e 100644 --- a/lib/gitlab/alert_management/payload/managed_prometheus.rb +++ b/lib/gitlab/alert_management/payload/managed_prometheus.rb @@ -35,18 +35,6 @@ module Gitlab gitlab_alert&.environment || super end - def metrics_dashboard_url - return unless gitlab_alert - - metrics_dashboard_project_prometheus_alert_url( - project, - gitlab_alert.prometheus_metric_id, - environment_id: environment.id, - embedded: true, - **alert_embed_window_params - ) - end - private def plain_gitlab_fingerprint diff --git a/lib/gitlab/alert_management/payload/prometheus.rb b/lib/gitlab/alert_management/payload/prometheus.rb index 76f3da8366b..15fa91646c8 100644 --- a/lib/gitlab/alert_management/payload/prometheus.rb +++ b/lib/gitlab/alert_management/payload/prometheus.rb @@ -78,18 +78,6 @@ module Gitlab rescue URI::InvalidURIError, KeyError end - def metrics_dashboard_url - return unless environment && full_query && title - - metrics_dashboard_project_environment_url( - project, - environment, - embed_json: dashboard_json, - embedded: true, - **alert_embed_window_params - ) - end - def has_required_attributes? project && title && starts_at_raw end @@ -108,29 +96,6 @@ module Gitlab def plain_gitlab_fingerprint [starts_at_raw, title, full_query].join('/') end - - # Formatted for parsing by JS - def alert_embed_window_params - { - start: (starts_at - METRIC_TIME_WINDOW).utc.strftime('%FT%TZ'), - end: (starts_at + METRIC_TIME_WINDOW).utc.strftime('%FT%TZ') - } - end - - def dashboard_json - { - panel_groups: [{ - panels: [{ - type: 'area-chart', - title: title, - y_label: gitlab_y_label, - metrics: [{ - query_range: full_query - }] - }] - }] - }.to_json - end end end end diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 0ea52b7b7c8..67fc2ae2fcc 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -47,11 +47,16 @@ module Gitlab Attribute.new(:root_caller_id, String), Attribute.new(:merge_action_status, String) ].freeze + private_constant :APPLICATION_ATTRIBUTES def self.known_keys KNOWN_KEYS end + def self.application_attributes + APPLICATION_ATTRIBUTES + end + def self.with_context(args, &block) application_context = new(**args) application_context.use(&block) @@ -79,12 +84,13 @@ module Gitlab end def initialize(**args) - unknown_attributes = args.keys - APPLICATION_ATTRIBUTES.map(&:name) + unknown_attributes = args.keys - self.class.application_attributes.map(&:name) raise ArgumentError, "#{unknown_attributes} are not known keys" if unknown_attributes.any? @set_values = args.keys assign_attributes(args) + set_attr_readers end # rubocop: disable Metrics/CyclomaticComplexity @@ -122,12 +128,14 @@ module Gitlab attr_reader :set_values - APPLICATION_ATTRIBUTES.each do |attr| - lazy_attr_reader attr.name, type: attr.type + def set_attr_readers + self.class.application_attributes.each do |attr| + self.class.lazy_attr_reader attr.name, type: attr.type + end end def assign_hash_if_value(hash, attribute_name) - unless KNOWN_KEYS.include?(attribute_name) + unless self.class.known_keys.include?(attribute_name) raise ArgumentError, "unknown attribute `#{attribute_name}`" end @@ -137,7 +145,7 @@ module Gitlab end def assign_attributes(values) - values.slice(*APPLICATION_ATTRIBUTES.map(&:name)).each do |name, value| + values.slice(*self.class.application_attributes.map(&:name)).each do |name, value| instance_variable_set("@#{name}", value) end end diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index 83d94d168a0..1bb92b7fa62 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -9,7 +9,8 @@ module Gitlab API_SCOPE = :api READ_API_SCOPE = :read_api READ_USER_SCOPE = :read_user - API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE].freeze + CREATE_RUNNER_SCOPE = :create_runner + API_SCOPES = [API_SCOPE, READ_API_SCOPE, READ_USER_SCOPE, CREATE_RUNNER_SCOPE].freeze PROFILE_SCOPE = :profile EMAIL_SCOPE = :email @@ -236,6 +237,10 @@ module Gitlab user.can?(:read_project, project) end + def bot_user_can_read_project?(user, project) + (user.project_bot? || user.security_policy_bot?) && can_read_project?(user, project) + end + def valid_oauth_token?(token) token && token.accessible? && valid_scoped_token?(token, Doorkeeper.configuration.scopes) end @@ -251,7 +256,8 @@ module Gitlab read_registry: [:read_container_image], write_registry: [:create_container_image], read_repository: [:download_code], - write_repository: [:download_code, :push_code] + write_repository: [:download_code, :push_code], + create_runner: [:create_instance_runner, :create_runner] } scopes.flat_map do |scope| @@ -316,7 +322,7 @@ module Gitlab return unless build.project.builds_enabled? if build.user - return unless build.user.can_log_in_with_non_expired_password? || (build.user.project_bot? && can_read_project?(build.user, build.project)) + return unless build.user.can_log_in_with_non_expired_password? || bot_user_can_read_project?(build.user, build.project) # If user is assigned to build, use restricted credentials of user Gitlab::Auth::Result.new(build.user, build.project, :build, build_authentication_abilities) diff --git a/lib/gitlab/auth/auth_finders.rb b/lib/gitlab/auth/auth_finders.rb index 4a610b26290..966520655a5 100644 --- a/lib/gitlab/auth/auth_finders.rb +++ b/lib/gitlab/auth/auth_finders.rb @@ -30,6 +30,7 @@ module Gitlab DEPLOY_TOKEN_HEADER = 'HTTP_DEPLOY_TOKEN' RUNNER_TOKEN_PARAM = :token RUNNER_JOB_TOKEN_PARAM = :token + PATH_DEPENDENT_FEED_TOKEN_REGEX = /\A#{User::FEED_TOKEN_PREFIX}(\h{64})-(\d+)\z/ # Check the Rails session for valid authentication details def find_user_from_warden @@ -54,7 +55,7 @@ module Gitlab token = current_request.params[:feed_token].presence || current_request.params[:rss_token].presence return unless token - User.find_by_feed_token(token) || raise(UnauthorizedError) + find_feed_token_user(token) || raise(UnauthorizedError) end def find_user_from_bearer_token @@ -195,6 +196,8 @@ module Gitlab when AccessTokenValidationService::EXPIRED raise ExpiredError when AccessTokenValidationService::REVOKED + revoke_token_family(access_token) + raise RevokedError when AccessTokenValidationService::IMPERSONATION_DISABLED raise ImpersonationDisabled @@ -277,6 +280,30 @@ module Gitlab PersonalAccessToken.find_by_token(password) end + def find_feed_token_user(token) + find_user_from_path_feed_token(token) || User.find_by_feed_token(token) + end + + def find_user_from_path_feed_token(token) + glft = token.match(PATH_DEPENDENT_FEED_TOKEN_REGEX) + + return unless glft + + # make sure that user id uses decimal notation + user_id = glft[2].to_i(10) + digest = glft[1] + + user = User.find_by_id(user_id) + return unless user + + feed_token = user.feed_token + our_digest = OpenSSL::HMAC.hexdigest("SHA256", feed_token, current_request.path) + + return unless ActiveSupport::SecurityUtils.secure_compare(digest, our_digest) + + user + end + def parsed_oauth_token Doorkeeper::OAuth::Token.from_request(current_request, *Doorkeeper.configuration.access_token_methods) end @@ -374,6 +401,12 @@ module Gitlab raise UnauthorizedError unless job end end + + def revoke_token_family(token) + return unless Feature.enabled?(:pat_reuse_detection) + + PersonalAccessTokens::RevokeTokenFamilyService.new(token).execute + end end end end diff --git a/lib/gitlab/auth/ldap/config.rb b/lib/gitlab/auth/ldap/config.rb index 30896637eff..13ca4f01154 100644 --- a/lib/gitlab/auth/ldap/config.rb +++ b/lib/gitlab/auth/ldap/config.rb @@ -264,7 +264,7 @@ module Gitlab return {} unless options['tls_options'] # Dup so we don't overwrite the original value - custom_options = options['tls_options'].dup.delete_if { |_, value| value.nil? || value.blank? } + custom_options = options['tls_options'].to_hash.delete_if { |_, value| value.nil? || value.blank? } custom_options.symbolize_keys! if custom_options[:cert] diff --git a/lib/gitlab/background_migration/.rubocop.yml b/lib/gitlab/background_migration/.rubocop.yml index 116c84c3759..9424686340f 100644 --- a/lib/gitlab/background_migration/.rubocop.yml +++ b/lib/gitlab/background_migration/.rubocop.yml @@ -59,3 +59,8 @@ Migration/BackgroundMigrationBaseClass: - 'base_job.rb' - 'batched_migration_job.rb' - 'logger.rb' + +BackgroundMigration/AvoidSilentRescueExceptions: + Enabled: true + Description: >- + Rescuing errors in batched background migration jobs can lead to undesired results diff --git a/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb b/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb new file mode 100644 index 00000000000..e3ad63aac2e --- /dev/null +++ b/lib/gitlab/background_migration/backfill_missing_ci_cd_settings.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # backfills project_ci_cd_settings + class BackfillMissingCiCdSettings < BatchedMigrationJob + # migrations only version of `project_ci_cd_settings` table + class ProjectCiCdSetting < ::ApplicationRecord + self.table_name = 'project_ci_cd_settings' + end + + operation_name :backfill_missing_ci_cd_settings + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch = sub_batch.where(%{ + NOT EXISTS ( + SELECT 1 + FROM project_ci_cd_settings + WHERE project_ci_cd_settings.project_id = projects.id + ) + }) + next unless sub_batch.present? + + ci_cd_attributes = sub_batch.map do |project| + { + project_id: project.id, + default_git_depth: 20, + forward_deployment_enabled: true + } + end + + ProjectCiCdSetting.insert_all(ci_cd_attributes) + end + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb b/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb new file mode 100644 index 00000000000..4dccd3fd852 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_uuid_conversion_column_in_vulnerability_occurrences.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This batched background migration will backfill values for `uuid_convert_string_to_uuid` column in + # vulnerability_occurrences table to allow us to migrate the column type from `varchar(36)` to `uuid` + class BackfillUuidConversionColumnInVulnerabilityOccurrences < BatchedMigrationJob + operation_name :backfill_uuid_conversion_column_in_vulnerability_occurrences + scope_to ->(relation) do + relation.where("uuid_convert_string_to_uuid = '00000000-0000-0000-0000-000000000000'::uuid") + end + feature_category :vulnerability_management + + def perform + each_sub_batch do |sub_batch| + sub_batch.update_all("uuid_convert_string_to_uuid = uuid::uuid") + end + end + end + end +end diff --git a/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb b/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb new file mode 100644 index 00000000000..2672498b627 --- /dev/null +++ b/lib/gitlab/background_migration/redis/backfill_project_pipeline_status_ttl.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + module Redis + # BackfillProjectPipelineStatusTtl cleans up keys written by + # Gitlab::Cache::Ci::ProjectPipelineStatus by adding a minimum 8-hour ttl + # to all keys. This either sets or extends the ttl of matching keys. + # + class BackfillProjectPipelineStatusTtl # rubocop:disable Migration/BackgroundMigrationBaseClass + def perform(keys) + # spread out deletes over a 4 hour period starting in 8 hours time + ttl_duration = 10.hours.to_i + ttl_jitter = 2.hours.to_i + + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + keys.each { |key| pipeline.expire(key, ttl_duration + rand(-ttl_jitter..ttl_jitter)) } + end + end + end + + def scan_match_pattern + "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}:project:*:pipeline_status" + end + + def redis + @redis ||= ::Redis.new(Gitlab::Redis::Cache.params) + end + end + end + end +end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index b2630a7ad7a..4beb8f54abf 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -9,6 +9,8 @@ module Gitlab class ProjectPipelineStatus include Gitlab::Utils::StrongMemoize + STATUS_KEY_TTL = 8.hours + attr_accessor :sha, :status, :ref, :project, :loaded def self.load_for_project(project) @@ -89,12 +91,17 @@ module Gitlab self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) self.status = nil if self.status.empty? + + redis.expire(cache_key, STATUS_KEY_TTL) end end def store_in_cache with_redis do |redis| - redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + redis.pipelined do |p| + p.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) + p.expire(cache_key, STATUS_KEY_TTL) + end end end diff --git a/lib/gitlab/cache/client.rb b/lib/gitlab/cache/client.rb index 37d6cac8d43..1e2962a5151 100644 --- a/lib/gitlab/cache/client.rb +++ b/lib/gitlab/cache/client.rb @@ -5,61 +5,40 @@ module Gitlab # It replaces Rails.cache with metrics support class Client DEFAULT_BACKING_RESOURCE = :unknown + DEFAULT_FEATURE_CATEGORY = :not_owned - # Build Cache client with the metadata support - # - # @param cache_identifier [String] defines the location of the cache definition - # Example: "ProtectedBranches::CacheService#fetch" - # @param feature_category [Symbol] name of the feature category (from config/feature_categories.yml) - # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) - # @return [Gitlab::Cache::Client] - def self.build_with_metadata( - cache_identifier:, - feature_category:, - backing_resource: DEFAULT_BACKING_RESOURCE - ) - new(Metadata.new( - cache_identifier: cache_identifier, - feature_category: feature_category, - backing_resource: backing_resource - )) - end - - def initialize(metadata, backend: Rails.cache) - @metadata = metadata - @metrics = Metrics.new(metadata) + def initialize(metrics, backend: Rails.cache) + @metrics = metrics @backend = backend end - def read(name) - read_result = backend.read(name) + def read(name, options = nil, labels = {}) + read_result = backend.read(name, options) if read_result.nil? - metrics.increment_cache_miss + metrics.increment_cache_miss(labels) else - metrics.increment_cache_hit + metrics.increment_cache_hit(labels) end read_result end - def fetch(name, options = nil, &block) - read_result = read(name) + def fetch(name, options = nil, labels = {}, &block) + read_result = read(name, options, labels) return read_result unless block || read_result backend.fetch(name, options) do - metrics.observe_cache_generation(&block) + metrics.observe_cache_generation(labels, &block) end end delegate :write, :exist?, :delete, to: :backend - attr_reader :metadata, :metrics - private - attr_reader :backend + attr_reader :metrics, :backend end end end diff --git a/lib/gitlab/cache/metadata.rb b/lib/gitlab/cache/metadata.rb index de35b332300..03ee48399d9 100644 --- a/lib/gitlab/cache/metadata.rb +++ b/lib/gitlab/cache/metadata.rb @@ -12,12 +12,12 @@ module Gitlab # @param backing_resource [Symbol] most affected resource by cache generation (full list: VALID_BACKING_RESOURCES) # @return [Gitlab::Cache::Metadata] def initialize( - cache_identifier:, - feature_category:, + cache_identifier: nil, + feature_category: Client::DEFAULT_FEATURE_CATEGORY, backing_resource: Client::DEFAULT_BACKING_RESOURCE ) @cache_identifier = cache_identifier - @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @feature_category = fetch_feature_category!(feature_category) @backing_resource = fetch_backing_resource!(backing_resource) end @@ -25,6 +25,12 @@ module Gitlab private + def fetch_feature_category!(feature_category) + return feature_category if feature_category == Client::DEFAULT_FEATURE_CATEGORY + + Gitlab::FeatureCategories.default.get!(feature_category) + end + def fetch_backing_resource!(resource) return resource if VALID_BACKING_RESOURCES.include?(resource) diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb index d9c80f076b9..26a1f346f13 100644 --- a/lib/gitlab/cache/metrics.rb +++ b/lib/gitlab/cache/metrics.rb @@ -12,14 +12,14 @@ module Gitlab # Increase cache hit counter # - def increment_cache_hit - counter.increment(labels.merge(cache_hit: true)) + def increment_cache_hit(labels = {}) + counter.increment(base_labels.merge(labels, cache_hit: true)) end # Increase cache miss counter # - def increment_cache_miss - counter.increment(labels.merge(cache_hit: false)) + def increment_cache_miss(labels = {}) + counter.increment(base_labels.merge(labels, cache_hit: false)) end # Measure the duration of cacheable action @@ -29,12 +29,12 @@ module Gitlab # cacheable_action # end # - def observe_cache_generation(&block) + def observe_cache_generation(labels = {}, &block) real_start = Gitlab::Metrics::System.monotonic_time value = yield - histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + histogram.observe(base_labels.merge(labels), Gitlab::Metrics::System.monotonic_time - real_start) value end @@ -44,20 +44,24 @@ module Gitlab attr_reader :cache_metadata def counter - @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") + @counter ||= Gitlab::Metrics.counter( + :redis_hit_miss_operations_total, + "Hit/miss Redis cache counter", + base_labels + ) end def histogram @histogram ||= Gitlab::Metrics.histogram( :redis_cache_generation_duration_seconds, 'Duration of Redis cache generation', - labels, + base_labels, DEFAULT_BUCKETS ) end - def labels - @labels ||= { + def base_labels + @base_labels ||= { cache_identifier: cache_metadata.cache_identifier, feature_category: cache_metadata.feature_category, backing_resource: cache_metadata.backing_resource diff --git a/lib/gitlab/checks/changes_access.rb b/lib/gitlab/checks/changes_access.rb index 194e3f6e938..3fd7e44985e 100644 --- a/lib/gitlab/checks/changes_access.rb +++ b/lib/gitlab/checks/changes_access.rb @@ -117,6 +117,7 @@ module Gitlab def bulk_access_checks! Gitlab::Checks::LfsCheck.new(self).validate! + Gitlab::Checks::GlobalFileSizeCheck.new(self).validate! end def blank_rev?(rev) diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 1186b532baf..bce4f969284 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -74,7 +74,7 @@ module Gitlab lfs_lock = project.lfs_file_locks.where(path: paths).where.not(user_id: user_access.user.id).take if lfs_lock - return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.name}" + return "The path '#{lfs_lock.path}' is locked in Git LFS by #{lfs_lock.user.username}" end end end diff --git a/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb new file mode 100644 index 00000000000..78f1716274e --- /dev/null +++ b/lib/gitlab/checks/file_size_check/allow_existing_oversized_blobs.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module FileSizeCheck + class AllowExistingOversizedBlobs + def initialize(project:, changes:, file_size_limit_megabytes:) + @project = project + @changes = changes + @oldrevs = changes.pluck(:oldrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @file_size_limit_megabytes = file_size_limit_megabytes + end + + def find(timeout: nil) + oversize_blobs = any_oversize_blobs.find(timeout: timeout) + + return oversize_blobs unless oldrevs.present? + + revs_paths = oldrevs.product(oversize_blobs.map(&:path)) + existing_blobs = project.repository.blobs_at(revs_paths, blob_size_limit: 1) + map_existing_path_to_size = existing_blobs.group_by(&:path).transform_values { |blobs| blobs.map(&:size).max } + + # return blobs that are going to be over the limit that were previously within the limit + oversize_blobs.select { |blob| map_existing_path_to_size.fetch(blob.path, 0) <= file_size_limit_megabytes } + end + + private + + attr_reader :project, :changes, :newrevs, :oldrevs, :file_size_limit_megabytes + + def any_oversize_blobs + AnyOversizedBlobs.new(project: project, changes: changes, + file_size_limit_megabytes: file_size_limit_megabytes) + end + end + end + end +end diff --git a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb new file mode 100644 index 00000000000..35f969dbb46 --- /dev/null +++ b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + module FileSizeCheck + class AnyOversizedBlobs + def initialize(project:, changes:, file_size_limit_megabytes:) + @project = project + @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @file_size_limit_megabytes = file_size_limit_megabytes + end + + def find(timeout: nil) + blobs = project.repository.new_blobs(newrevs, dynamic_timeout: timeout) + + blobs.select do |blob| + ::Gitlab::Utils.bytes_to_megabytes(blob.size) > file_size_limit_megabytes + end + end + + private + + attr_reader :project, :newrevs, :file_size_limit_megabytes + end + end + end +end diff --git a/lib/gitlab/checks/global_file_size_check.rb b/lib/gitlab/checks/global_file_size_check.rb new file mode 100644 index 00000000000..418d2d32b57 --- /dev/null +++ b/lib/gitlab/checks/global_file_size_check.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module Checks + class GlobalFileSizeCheck < BaseBulkChecker + MAX_FILE_SIZE_MB = 100 + LOG_MESSAGE = 'Checking for blobs over the file size limit' + + def validate! + return unless Feature.enabled?(:global_file_size_check, project) + + Gitlab::AppJsonLogger.info(LOG_MESSAGE) + logger.log_timed(LOG_MESSAGE) do + Gitlab::Checks::FileSizeCheck::AllowExistingOversizedBlobs.new( + project: project, + changes: changes, + file_size_limit_megabytes: MAX_FILE_SIZE_MB + ).find + + # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/393535 + # - set limit per plan tier + # - raise an error if large blobs are found + end + + true + end + end + end +end diff --git a/lib/gitlab/ci/artifact_file_reader.rb b/lib/gitlab/ci/artifact_file_reader.rb index 2eb8df01d58..0d8f6f3ea40 100644 --- a/lib/gitlab/ci/artifact_file_reader.rb +++ b/lib/gitlab/ci/artifact_file_reader.rb @@ -14,7 +14,8 @@ module Gitlab def initialize(job) @job = job - raise ArgumentError, 'Job does not have artifacts' unless @job.artifacts? + raise Error, 'Job doesnt exist' unless @job + raise Error, 'Job does not have artifacts' unless @job.artifacts? validate! end diff --git a/lib/gitlab/ci/build/rules.rb b/lib/gitlab/ci/build/rules.rb index 17b9f30db33..8b503290e6e 100644 --- a/lib/gitlab/ci/build/rules.rb +++ b/lib/gitlab/ci/build/rules.rb @@ -32,18 +32,13 @@ module Gitlab if @rule_list.nil? Result.new(when: @default_when) elsif matched_rule = match_rule(pipeline, context) - result = Result.new( + Result.new( when: matched_rule.attributes[:when] || @default_when, start_in: matched_rule.attributes[:start_in], allow_failure: matched_rule.attributes[:allow_failure], - variables: matched_rule.attributes[:variables] + variables: matched_rule.attributes[:variables], + needs: matched_rule.attributes[:needs] ) - - if Feature.enabled?(:introduce_rules_with_needs, pipeline.project) - result.needs = matched_rule.attributes[:needs] - end - - result else Result.new(when: 'never') end diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index 27a7611ffdd..e0ef598da1b 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -73,9 +73,7 @@ module Gitlab end def latest_version_sha - return unless catalog_resource = project&.catalog_resource - - catalog_resource.latest_version&.sha + project.releases.latest&.sha end end end diff --git a/lib/gitlab/ci/config/README.md b/lib/gitlab/ci/config/README.md new file mode 100644 index 00000000000..e850afc5253 --- /dev/null +++ b/lib/gitlab/ci/config/README.md @@ -0,0 +1,178 @@ +# `::Gitlab::Ci::Config` module overview + +`::Gitlab::Ci::Config` is a concrete implementation of abstract +`::Gitlab::Config` module. It's being used to build, traverse and translate +hierarchical, user-provided, CI configuration, usually provided in +`.gitlab-ci.yml` and included files. + +## High-level Overview + +`::Gitlab::Ci::Config` is an indirection layer between user-provided data and +GitLab itself. + +1. A user provides YAML configuration in `.gitlab-ci.yml` and all included files. +1. `::Gitlab::Ci::Config` loads the provided YAML using Ruby standard `Psych` library. +1. The resulting Hash is then passed to the module to build an Abstract Syntax Tree. +1. The module validates, transforms, translates and augments the data to build + a stable representation of user-provided configuration. + +This additional layer helps us to validate the user-provided configuration and +surface any errors to a user if it is not valid. In case of a valid +configuration, it makes it possible to build a stable representation of +config that we can depend on. + +For example, both following configurations using the +[environment](https://docs.gitlab.com/ee/ci/yaml/#environment) +keyword are correct: + +```yaml +# First way to define an environment: + +deploy: + environment: production + script: cap deploy + +# Second way to define an environment: + +deploy: + environment: + name: production + url: https://prod.example.com + kubernetes: + namespace: production +``` + +This demonstrates the concept of hidden / expanding complexity: if users need +more flexibility, they can opt-in into using a much more elaborate syntax to +configure their environments. **We use this technique to make it possible for +simplicity to coexist with flexibility without additional complexity**. + +`::Gitlab::Ci::Config` allows us to achieve this, because it is an indirection +layer, that translates user-provided configuration into a known and expected +format when users can achieve the same thing in `.gitlab-ci.yml` in a few +different ways. + +## Hierarchical configuration + +`.gitlab-ci.yml` configuration is hierarchical but same keywords can often be +used on different levels in the hierarchy. `::Gitlab::Ci::Config` module makes +it easier to manage the complexity that stems from having same keyword +available in [many different places](https://docs.gitlab.com/ee/ci/yaml/#default): + +```yaml +default: + image: ruby:3.0 + +rspec: + script: bundle exec rspec + +rspec 2.7: + image: ruby:2.7 + script: bundle exec rspec +``` + +We can achieve that, because in `::Gitlab::Ci::Config` most of the keywords are +implemented within separate Ruby classes, that then can be reused: + +```ruby +# Simplified version of an entry class that describes a Docker image. +# +class Gitlab::Ci::Config::Entry + class Image < ::Gitlab::Config::Entry::Node + + validates :config, allowed_keys: ALLOWED_IMAGE_CONFIG_KEYS + + def value + if string? + { name: @config } + elsif hash? + { + name: @config[:name], + entrypoint: @config[:entrypoint], + ports: (ports_value if ports_defined?), + pull_policy: pull_policy_value + } + else + {} + end + end + end +end +``` + +The config above is a simple demonstration of the translation layer, into a +stable configuration, depending on what simplification strategy has been used +by a user. There more complex examples, though: + +```ruby +module Gitlab::Ci::Config::Entry + class Need < ::Gitlab::Config::Entry::Simplifiable + strategy :JobString, if: -> (config) { config.is_a?(String) } + + strategy :JobHash, + if: -> (config) { config.is_a?(Hash) && same_pipeline_need?(config) } + + strategy :CrossPipelineDependency, + if: -> (config) { config.is_a?(Hash) && cross_pipeline_need?(config) } + + # [ ... ] + end +end +``` + +Every time we load config, an Abstract Syntax Tree is being built, because +nodes / entries know what the child nodes can be: + +```ruby +# Simplified root entry code +# +module Gitlab::Ci::Config::Entry + class Root < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Configurable + + entry :default, Entry::Default, + description: 'Default configuration for all jobs.' + + entry :include, Entry::Includes, + description: 'List of external YAML files to include.' + + entry :before_script, Entry::Commands, + description: 'Script that will be executed before each job.' + + entry :image, Entry::Image, + description: 'Docker image that will be used to execute jobs.' + + entry :services, Entry::Services, + description: 'Docker images that will be linked to the container.' + + entry :after_script, Entry::Commands, + description: 'Script that will be executed after each job.' + + entry :variables, Entry::Variables, + description: 'Environment variables that will be used.' + + # [ ... ] + end +end +``` + +Loading the configuration script mentioned at the beginning of this pargraph +will result in build a following AST: + +``` +Entry::Root +`- + |- Entry::Default + | `- Entry::Image('ruby:3.0') + | + |- Entry::Job('rspec') + | `- Entry::Script('bundle exec rspec') + | + |- Entry::Job('rspec 2.7') + | |- Entry::Image('ruby:2.7) + | `- Entry::Script('bundle exec rspec') +``` + +The AST will be validated, and eventually will generate a stable representation +of configuration that we can use to persist pipelines / stages / jobs in the +database, and start pipeline processing. diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 273d78bd583..f23fa2e6401 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -19,12 +19,15 @@ module Gitlab end def content - strong_memoize(:content) do - Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) - rescue Gitlab::Ci::ArtifactFileReader::Error => error - errors.push(error.message) # TODO this memoizes the error message as a content! - end + return unless context.parent_pipeline.present? + + Gitlab::Ci::ArtifactFileReader.new(artifact_job).read(location) + rescue Gitlab::Ci::ArtifactFileReader::Error => error + errors.push(error.message) + + nil end + strong_memoize_attr :content def metadata super.merge( diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 61d95c8d4e6..8bcb2a389d2 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -90,15 +90,7 @@ module Gitlab end def load_and_validate_expanded_hash! - context.logger.instrument(:config_file_fetch_content_hash) do - content_result # calling the method loads YAML then memoizes the content result - end - - context.logger.instrument(:config_file_interpolate_result) do - interpolator.interpolate! - end - - return validate_interpolation! unless interpolator.valid? + return errors.push("`#{masked_location}`: #{content_result.error}") unless content_result.valid? context.logger.instrument(:config_file_expand_content_includes) do expanded_content_hash # calling the method expands then memoizes the result @@ -109,36 +101,24 @@ module Gitlab protected - def content_result - ::Gitlab::Ci::Config::Yaml - .load_result!(content, project: context.project) - end - strong_memoize_attr :content_result - def content_inputs # TODO: remove support for `with` syntax in 16.1, see https://gitlab.com/gitlab-org/gitlab/-/issues/408369 # In the interim prefer `inputs` over `with` while allow either syntax. params.to_h.slice(:inputs, :with).each_value.first end - strong_memoize_attr :content_inputs - - def content_hash - interpolator.interpolate! - interpolator.to_hash - end - strong_memoize_attr :content_hash - - def interpolator - Yaml::Interpolator.new(content_result, content_inputs, context) + def content_result + context.logger.instrument(:config_file_fetch_content_hash) do + ::Gitlab::Ci::Config::Yaml::Loader.new(content, inputs: content_inputs, current_user: context.user).load + end end - strong_memoize_attr :interpolator + strong_memoize_attr :content_result def expanded_content_hash - return if content_hash.blank? + return if content_result.content.blank? strong_memoize(:expanded_content_hash) do - expand_includes(content_hash) + expand_includes(content_result.content) end end @@ -148,12 +128,6 @@ module Gitlab end end - def validate_interpolation! - return if interpolator.valid? - - errors.push("`#{masked_location}`: #{interpolator.error_message}") - end - def expand_includes(hash) External::Processor.new(hash, context.mutate(expand_context_attrs)).perform end diff --git a/lib/gitlab/ci/config/external/rules.rb b/lib/gitlab/ci/config/external/rules.rb index 134306332e6..59e666b8bb5 100644 --- a/lib/gitlab/ci/config/external/rules.rb +++ b/lib/gitlab/ci/config/external/rules.rb @@ -17,16 +17,12 @@ module Gitlab end def evaluate(context) - if Feature.enabled?(:ci_support_include_rules_when_never, context.project) - if @rule_list.nil? - Result.new('always') - elsif matched_rule = match_rule(context) - Result.new(matched_rule.attributes[:when]) - else - Result.new('never') - end + if @rule_list.nil? + Result.new('always') + elsif matched_rule = match_rule(context) + Result.new(matched_rule.attributes[:when]) else - LegacyResult.new(@rule_list.nil? || match_rule(context)) + Result.new('never') end end @@ -55,12 +51,6 @@ module Gitlab self.when != 'never' end end - - LegacyResult = Struct.new(:result) do - def pass? - !!result - end - end end end end diff --git a/lib/gitlab/ci/config/yaml.rb b/lib/gitlab/ci/config/yaml.rb index f74ef95a832..e3010ac3fdb 100644 --- a/lib/gitlab/ci/config/yaml.rb +++ b/lib/gitlab/ci/config/yaml.rb @@ -4,21 +4,17 @@ module Gitlab module Ci class Config module Yaml + LoadError = Class.new(StandardError) + class << self - def load!(content, project: nil) - Loader.new(content, project: project).to_result.then do |result| - ## - # raise an error for backwards compatibility - # - raise result.error unless result.valid? + def load!(content, current_user: nil) + Loader.new(content, current_user: current_user).load.then do |result| + raise result.error_class, result.error if !result.valid? && result.error_class.present? + raise LoadError, result.error unless result.valid? result.content end end - - def load_result!(content, project: nil) - Loader.new(content, project: project).to_result - end end end end diff --git a/lib/gitlab/ci/config/yaml/interpolator.rb b/lib/gitlab/ci/config/yaml/interpolator.rb index 4ae191dfedf..2909c2ac798 100644 --- a/lib/gitlab/ci/config/yaml/interpolator.rb +++ b/lib/gitlab/ci/config/yaml/interpolator.rb @@ -5,42 +5,23 @@ module Gitlab class Config module Yaml ## - # Config::Yaml::Interpolation performs includable file interpolation, and surfaces all possible interpolation + # Config::Yaml::Interpolator performs CI config file interpolation, and surfaces all possible interpolation # errors. It is designed to provide an external file's validation context too. # class Interpolator - include ::Gitlab::Utils::StrongMemoize + attr_reader :config, :args, :current_user, :errors - attr_reader :config, :args, :ctx, :errors - - def initialize(config, args, ctx = nil) + def initialize(config, args, current_user: nil) @config = config @args = args.to_h - @ctx = ctx + @current_user = current_user @errors = [] - - validate! end def valid? @errors.none? end - def ready? - ## - # Interpolation is ready when it has been either interrupted by an error or finished with a result. - # - @result || @errors.any? - end - - def interpolate? - enabled? && has_header? && valid? - end - - def has_header? - config.has_header? && config.header.present? - end - def to_hash @result.to_h end @@ -55,43 +36,25 @@ module Gitlab @errors.first(3).join(', ') end - ## - # TODO Add `instrument.logger` instrumentation blocks: - # https://gitlab.com/gitlab-org/gitlab/-/issues/396722 - # def interpolate! - return {} unless valid? - return @result ||= content.to_h unless interpolate? + return @errors.push(config.error) unless config.valid? + return @result ||= config.content unless config.has_header? return @errors.concat(header.errors) unless header.valid? return @errors.concat(inputs.errors) unless inputs.valid? return @errors.concat(context.errors) unless context.valid? return @errors.concat(template.errors) unless template.valid? - if ctx&.user - ::Gitlab::UsageDataCounters::HLLRedisCounter.track_event('ci_interpolation_users', values: ctx.user.id) + if current_user.present? + ::Gitlab::UsageDataCounters::HLLRedisCounter + .track_event('ci_interpolation_users', values: current_user.id) end @result ||= template.interpolated.to_h.deep_symbolize_keys end - strong_memoize_attr :interpolate! private - def validate! - return errors.push('content does not have a valid YAML syntax') unless config.valid? - - return unless has_header? && !enabled? - - errors.push('can not evaluate included file because interpolation is disabled') - end - - def enabled? - return false if ctx.nil? - - ::Feature.enabled?(:ci_includable_files_interpolation, ctx.project) - end - def header @entry ||= Ci::Config::Header::Root.new(config.header).tap do |header| header.key = 'header' diff --git a/lib/gitlab/ci/config/yaml/loader.rb b/lib/gitlab/ci/config/yaml/loader.rb index 924a1f2e46b..fb24a2874e4 100644 --- a/lib/gitlab/ci/config/yaml/loader.rb +++ b/lib/gitlab/ci/config/yaml/loader.rb @@ -5,33 +5,45 @@ module Gitlab class Config module Yaml class Loader + include Gitlab::Utils::StrongMemoize + AVAILABLE_TAGS = [Config::Yaml::Tags::Reference].freeze MAX_DOCUMENTS = 2 - def initialize(content, project: nil) + def initialize(content, inputs: {}, current_user: nil) @content = content - @project = project + @current_user = current_user + @inputs = inputs end - def to_result - Yaml::Result.new(config: load!, error: nil) - rescue ::Gitlab::Config::Loader::FormatError => e - Yaml::Result.new(error: e) - end + def load + yaml_result = load_uninterpolated_yaml - private + return yaml_result unless yaml_result.valid? - attr_reader :content, :project + interpolator = Yaml::Interpolator.new(yaml_result, inputs, current_user: current_user) - def ensure_custom_tags - @ensure_custom_tags ||= begin - AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + interpolator.interpolate! - true + if interpolator.valid? + # This Result contains only the interpolated config and does not have a header + Yaml::Result.new(config: interpolator.to_hash, error: nil) + else + Yaml::Result.new(error: interpolator.error_message) end end - def load! + private + + attr_reader :content, :current_user, :inputs + + def load_uninterpolated_yaml + Yaml::Result.new(config: load_yaml!, error: nil) + rescue ::Gitlab::Config::Loader::FormatError => e + Yaml::Result.new(error: e.message, error_class: e) + end + + def load_yaml! ensure_custom_tags ::Gitlab::Config::Loader::MultiDocYaml.new( @@ -41,6 +53,14 @@ module Gitlab reject_empty: true ).load! end + + def ensure_custom_tags + @ensure_custom_tags ||= begin + AVAILABLE_TAGS.each { |klass| Psych.add_tag(klass.tag, klass) } + + true + end + end end end end diff --git a/lib/gitlab/ci/config/yaml/result.rb b/lib/gitlab/ci/config/yaml/result.rb index 6b53adc3a57..6b20eeae203 100644 --- a/lib/gitlab/ci/config/yaml/result.rb +++ b/lib/gitlab/ci/config/yaml/result.rb @@ -5,11 +5,12 @@ module Gitlab class Config module Yaml class Result - attr_reader :error + attr_reader :error, :error_class - def initialize(config: nil, error: nil) + def initialize(config: nil, error: nil, error_class: nil) @config = Array.wrap(config) @error = error + @error_class = error_class end def valid? diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 9e71a9e8e91..6ce662bdead 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -3,6 +3,8 @@ module Gitlab module Ci class JwtV2 < Jwt + include Gitlab::Utils::StrongMemoize + DEFAULT_AUD = Settings.gitlab.base_url GITLAB_HOSTED_RUNNER = 'gitlab-hosted' SELF_HOSTED_RUNNER = 'self-hosted' @@ -48,31 +50,35 @@ module Gitlab sha: pipeline.sha } - if Feature.enabled?(:ci_jwt_v2_ref_uri_claim, pipeline.project) + if project_config&.source == :repository_source additional_claims[:ci_config_ref_uri] = ci_config_ref_uri + additional_claims[:ci_config_sha] = pipeline.sha end super.merge(additional_claims) end def ci_config_ref_uri - project_config = Gitlab::Ci::ProjectConfig.new( + "#{project_config&.url}@#{pipeline.source_ref_path}" + rescue StandardError => e + # We don't want endpoints relying on this code to fail if there's an error here. + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) + nil + end + + def project_config + Gitlab::Ci::ProjectConfig.new( project: project, sha: pipeline.sha, pipeline_source: pipeline.source&.to_sym, pipeline_source_bridge: pipeline.source_bridge ) - - return unless project_config&.source == :repository_source - - "#{project_config.url}@#{pipeline.source_ref_path}" - - # Errors are rescued to mitigate risk. This can be removed if no errors are observed. - # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/117923#note_1387660746 for context. rescue StandardError => e + # We don't want endpoints relying on this code to fail if there's an error here. Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, pipeline_id: pipeline.id) nil end + strong_memoize_attr(:project_config) def runner_environment return unless runner diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 21408beb8cb..ee1da82f285 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -126,8 +126,8 @@ module Gitlab compare_key: data['cve'] || '', location: location, evidence: evidence, - severity: parse_severity_level(data['severity']), - confidence: parse_confidence_level(data['confidence']), + severity: ::Enums::Vulnerability.parse_severity_level(data['severity']), + confidence: ::Enums::Vulnerability.parse_confidence_level(data['confidence']), scanner: create_scanner(top_level_scanner_data || data['scanner']), scan: report&.scan, identifiers: identifiers, @@ -260,14 +260,6 @@ module Gitlab ::Gitlab::Ci::Reports::Security::Link.new(name: link['name'], url: link['url']) end - def parse_severity_level(input) - input&.downcase.then { |value| ::Enums::Vulnerability.severity_levels.key?(value) ? value : 'unknown' } - end - - def parse_confidence_level(input) - input&.downcase.then { |value| ::Enums::Vulnerability.confidence_levels.key?(value) ? value : 'unknown' } - end - def create_location(location_data) raise NotImplementedError end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 92d9d170575..e39482481c7 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6], - secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.6] + cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], + secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6] }.freeze VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..5563acbe232 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/cluster-image-scanning-report-format.json @@ -0,0 +1,1035 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json new file mode 100644 index 00000000000..820811100ea --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/container-scanning-report-format.json @@ -0,0 +1,967 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..63e39395772 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/coverage-fuzzing-report-format.json @@ -0,0 +1,925 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json new file mode 100644 index 00000000000..86e62558a39 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dast-report-format.json @@ -0,0 +1,1330 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json new file mode 100644 index 00000000000..c08cbcffc5b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/dependency-scanning-report-format.json @@ -0,0 +1,1033 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json new file mode 100644 index 00000000000..f1869950d20 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/sast-report-format.json @@ -0,0 +1,920 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json new file mode 100644 index 00000000000..e9bfd6186a3 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.5/secret-detection-report-format.json @@ -0,0 +1,944 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.5" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb index 035167f1a74..b8b70a6b6b6 100644 --- a/lib/gitlab/ci/pipeline/chain/validate/abilities.rb +++ b/lib/gitlab/ci/pipeline/chain/validate/abilities.rb @@ -22,7 +22,7 @@ module Gitlab return error('Insufficient permissions to create a new pipeline') end - unless allowed_to_write_ref? + unless allowed_to_run_pipeline? error("You do not have sufficient permission to run a pipeline on '#{command.ref}'. Please select a different branch or contact your administrator for assistance.") end end @@ -37,6 +37,10 @@ module Gitlab can?(current_user, :create_pipeline, project) end + def allowed_to_run_pipeline? + allowed_to_write_ref? + end + def allowed_to_write_ref? access = Gitlab::UserAccess.new(current_user, container: project) diff --git a/lib/gitlab/ci/project_config/bridge.rb b/lib/gitlab/ci/project_config/bridge.rb index c342ab2c215..45aa330508f 100644 --- a/lib/gitlab/ci/project_config/bridge.rb +++ b/lib/gitlab/ci/project_config/bridge.rb @@ -10,6 +10,11 @@ module Gitlab pipeline_source_bridge.yaml_for_downstream end + # Bridge.yaml_for_downstream injects an `include` + def internal_include_prepended? + true + end + def source :bridge_source end diff --git a/lib/gitlab/ci/project_config/repository.rb b/lib/gitlab/ci/project_config/repository.rb index 7dfd528fd6f..a08cf27b74c 100644 --- a/lib/gitlab/ci/project_config/repository.rb +++ b/lib/gitlab/ci/project_config/repository.rb @@ -4,6 +4,8 @@ module Gitlab module Ci class ProjectConfig class Repository < Source + extend ::Gitlab::Utils::Override + def content strong_memoize(:content) do next unless file_in_repository? diff --git a/lib/gitlab/ci/project_config/source.rb b/lib/gitlab/ci/project_config/source.rb index 68853ca8296..5f37c3bad7b 100644 --- a/lib/gitlab/ci/project_config/source.rb +++ b/lib/gitlab/ci/project_config/source.rb @@ -5,7 +5,6 @@ module Gitlab class ProjectConfig class Source include Gitlab::Utils::StrongMemoize - extend ::Gitlab::Utils::Override def initialize(project, sha, custom_content, pipeline_source, pipeline_source_bridge) @project = project diff --git a/lib/gitlab/ci/reports/sbom/source.rb b/lib/gitlab/ci/reports/sbom/source.rb index fbb8644c1b0..b7af6ea17c3 100644 --- a/lib/gitlab/ci/reports/sbom/source.rb +++ b/lib/gitlab/ci/reports/sbom/source.rb @@ -11,6 +11,22 @@ module Gitlab @source_type = type @data = data end + + def source_file_path + data.dig('source_file', 'path') + end + + def input_file_path + data.dig('input_file', 'path') + end + + def packager + data.dig('package_manager', 'name') + end + + def language + data.dig('language', 'name') + end end end end diff --git a/lib/gitlab/ci/reports/security/link.rb b/lib/gitlab/ci/reports/security/link.rb index 1c4c05cd9ac..6804d2b2a29 100644 --- a/lib/gitlab/ci/reports/security/link.rb +++ b/lib/gitlab/ci/reports/security/link.rb @@ -18,6 +18,10 @@ module Gitlab url: url }.compact end + + def ==(other) + name == other.name && url == other.url + end end end end diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 49d3c270bac..46d0b92b243 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 49d3c270bac..46d0b92b243 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.34.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.37.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index f4a13d61ba2..b1e498a9d09 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index c1a3daa7f5b..5a7e69b62d9 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index a3c7c6baf02..dac559db8d5 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.50.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.51.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index f16c28e7b60..87d0894b67a 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -20,18 +20,18 @@ cache: paths: - ${TF_ROOT}/.terraform/ -.terraform:fmt: &terraform_fmt +.terraform:fmt: stage: validate script: - gitlab-terraform fmt allow_failure: true -.terraform:validate: &terraform_validate +.terraform:validate: stage: validate script: - gitlab-terraform validate -.terraform:build: &terraform_build +.terraform:build: stage: build script: - gitlab-terraform plan @@ -46,7 +46,7 @@ cache: reports: terraform: ${TF_ROOT}/plan.json -.terraform:deploy: &terraform_deploy +.terraform:deploy: stage: deploy script: - gitlab-terraform apply @@ -56,7 +56,7 @@ cache: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual -.terraform:destroy: &terraform_destroy +.terraform:destroy: stage: cleanup script: - gitlab-terraform destroy diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 793030d302a..d2b929cf995 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -22,7 +22,7 @@ variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend -.terraform:fmt: &terraform_fmt +.terraform:fmt: stage: validate script: - gitlab-terraform fmt @@ -33,7 +33,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:validate: &terraform_validate +.terraform:validate: stage: validate script: - gitlab-terraform validate @@ -43,7 +43,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:build: &terraform_build +.terraform:build: stage: build script: - gitlab-terraform plan @@ -63,7 +63,7 @@ variables: when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. -.terraform:deploy: &terraform_deploy +.terraform:deploy: stage: deploy script: - gitlab-terraform apply @@ -73,7 +73,7 @@ variables: - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: manual -.terraform:destroy: &terraform_destroy +.terraform:destroy: stage: cleanup script: - gitlab-terraform destroy diff --git a/lib/gitlab/ci/templates/npm.gitlab-ci.yml b/lib/gitlab/ci/templates/npm.gitlab-ci.yml index fb0d300338b..ae2edd6f3fa 100644 --- a/lib/gitlab/ci/templates/npm.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/npm.gitlab-ci.yml @@ -37,9 +37,15 @@ publish: # Compare the version in package.json to all published versions. # If the package.json version has not yet been published, run `npm publish`. + # If $SIGSTORE_ID_TOKEN is set this template will generate a provenance + # document. For more information refer to the documentation: https://docs.gitlab.com/ee/ci/yaml/signing_examples/ - | if [[ "$(npm view ${NPM_PACKAGE_NAME} versions)" != *"'${NPM_PACKAGE_VERSION}'"* ]]; then - npm publish + if [[ -n "${SIGSTORE_ID_TOKEN}" ]]; then + npm publish --provenance + else + npm publish + fi echo "Successfully published version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} to GitLab's NPM registry: ${CI_PROJECT_URL}/-/packages" else echo "Version ${NPM_PACKAGE_VERSION} of ${NPM_PACKAGE_NAME} has already been published, so no new version has been published." diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index 9960a6fbdf5..f77ef262236 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -8,6 +8,21 @@ module Gitlab attr_reader :errors + def self.fabricate(input) + case input + when Array + new(input) + when Hash + new(input.map { |key, value| { key: key, value: value } }) + when Proc + fabricate(input.call) + when self + input + else + raise ArgumentError, "Unknown `#{input.class}` variable collection!" + end + end + def initialize(variables = [], errors = nil) @variables = [] @variables_by_key = Hash.new { |h, k| h[k] = [] } diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index 0fcf11121fa..73452d83bce 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -17,6 +17,10 @@ module Gitlab @variable = { key: key, value: value, public: public, file: file, masked: masked, raw: raw } end + def key + @variable.fetch(:key) + end + def value @variable.fetch(:value) end diff --git a/lib/gitlab/ci/variables/downstream/base.rb b/lib/gitlab/ci/variables/downstream/base.rb new file mode 100644 index 00000000000..6845ed4cc1b --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/base.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class Base + def initialize(context) + @context = context + end + + private + + attr_reader :context + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb new file mode 100644 index 00000000000..6690e9f1c1f --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/expandable_variable_generator.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class ExpandableVariableGenerator < Base + def for(item) + expanded_value = ::ExpandVariables.expand(item.value, context.all_bridge_variables) + + [{ key: item.key, value: expanded_value }] + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/generator.rb b/lib/gitlab/ci/variables/downstream/generator.rb new file mode 100644 index 00000000000..93c995cc918 --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/generator.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class Generator + include Gitlab::Utils::StrongMemoize + + Context = Struct.new(:all_bridge_variables, keyword_init: true) + + def initialize(bridge) + @bridge = bridge + + context = Context.new(all_bridge_variables: bridge.variables) + + @raw_variable_generator = RawVariableGenerator.new(context) + @expandable_variable_generator = ExpandableVariableGenerator.new(context) + end + + def calculate + calculate_downstream_variables + .reverse # variables priority + .uniq { |var| var[:key] } # only one variable key to pass + .reverse + end + + private + + attr_reader :bridge, :all_bridge_variables + + def calculate_downstream_variables + # The order of this list refers to the priority of the variables + # The variables added later takes priority. + downstream_yaml_variables + + downstream_pipeline_variables + + downstream_pipeline_schedule_variables + end + + def downstream_yaml_variables + return [] unless bridge.forward_yaml_variables? + + build_downstream_variables_from(bridge.yaml_variables) + end + + def downstream_pipeline_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_variables = bridge.pipeline_variables.to_a + build_downstream_variables_from(pipeline_variables) + end + + def downstream_pipeline_schedule_variables + return [] unless bridge.forward_pipeline_variables? + + pipeline_schedule_variables = bridge.pipeline_schedule_variables.to_a + build_downstream_variables_from(pipeline_schedule_variables) + end + + def build_downstream_variables_from(variables) + Gitlab::Ci::Variables::Collection.fabricate(variables).flat_map do |item| + if item.raw? + @raw_variable_generator.for(item) + else + @expandable_variable_generator.for(item) + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb b/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb new file mode 100644 index 00000000000..42c795b4398 --- /dev/null +++ b/lib/gitlab/ci/variables/downstream/raw_variable_generator.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Variables + module Downstream + class RawVariableGenerator < Base + def for(item) + [{ key: item.key, value: item.value, raw: true }] + end + end + end + end + end +end diff --git a/lib/gitlab/cleanup/remote_uploads.rb b/lib/gitlab/cleanup/remote_uploads.rb index 6cadb9424f7..5811b6223a3 100644 --- a/lib/gitlab/cleanup/remote_uploads.rb +++ b/lib/gitlab/cleanup/remote_uploads.rb @@ -17,6 +17,13 @@ module Gitlab return end + if bucket_prefix.present? + error_message = "Uploads are configured with a bucket prefix '#{bucket_prefix}'.\n" + error_message += "Unfortunately, prefixes are not supported for this Rake task.\n" + # At the moment, Fog does not provide a cloud-agnostic way of iterating through a bucket with a prefix. + raise error_message + end + logger.info "Looking for orphaned remote uploads to remove#{'. Dry run' if dry_run}..." each_orphan_file do |file| @@ -77,6 +84,10 @@ module Gitlab def configuration Gitlab.config.uploads.object_store end + + def bucket_prefix + configuration.bucket_prefix + end end end end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index b39d2a02f02..c6ce0aa6160 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../utils' # Gitlab::Utils +require 'gitlab/utils/all' # Gitlab::Utils module Gitlab module Cluster diff --git a/lib/gitlab/config/README.md b/lib/gitlab/config/README.md new file mode 100644 index 00000000000..355dbdc8cfe --- /dev/null +++ b/lib/gitlab/config/README.md @@ -0,0 +1,29 @@ +# `::Gitlab::Config` module overview + +`::Gitlab::Config` is an abstract module used to build, traverse and translate +any kind of hierarchical, user-provided configuration. + +The most complex and widely used implementation is `::Gitlab::Ci::Config` +facade class. Please see `lib/gitlab/ci/config/README.md` for more information +around how it works. + +## High-level Overview + +The main motivation behind how `::Gitlab::Config` and `::Gitlab::Ci::Config` +work is to build an indirection layer between complex user-provided +configuration and GitLab itself. This helps us to extend configuration keywords +in a backwards-compatible way, and make sure that validation and transformation +rules are encapsulated within domain classes, what significantly helps to +reduce cognitive load on Engineers working on that part of the codebase. + +`Gitlab::Config` is a tool to work with hierarchical configuration: + +1. First we parse YAML with Ruby standard library `Psych`. +1. The resulting hash is being used to initialize a concrete implementation of `Gitlab::Config`. +1. In `::Gitlab::Ci::Config` abstract classes from `::Gitlab::Config` have their implementations. +1. Each domain class represents one or a group of hierarchical YAML entries, like `job:artifacts`. +1. Each entry knows what subentires are supported and how to validate them. +1. Upon loading a configuration we build an abstract syntax tree, and validate configuration. +1. If there are errors, the module can surface them to a user. +1. In case of config being valid, the config gets translated and augmented. +1. The result is a consistent representation that we can depend on in other parts of the codebase. diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 8fec5cf3303..e1e9e4720bb 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,13 +12,14 @@ module Gitlab author_url = build_author_url(build.commit, commit) - data = { + { object_kind: 'build', ref: build.ref, tag: build.tag, before_sha: build.before_sha, sha: build.sha, + retries_count: build.retries_count, # TODO: should this be not prefixed with build_? # Leaving this way to have backward compatibility @@ -69,10 +70,6 @@ module Gitlab environment: build_environment(build) } - - data[:retries_count] = build.retries_count if Feature.enabled?(:job_webhook_retries_count, project) - - data end private diff --git a/lib/gitlab/data_builder/emoji.rb b/lib/gitlab/data_builder/emoji.rb new file mode 100644 index 00000000000..63562eca155 --- /dev/null +++ b/lib/gitlab/data_builder/emoji.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module DataBuilder + module Emoji + extend self + + def build(award_emoji, user, action) + project = award_emoji.awardable.project + data = build_base_data(project, user, award_emoji, action) + + if award_emoji.awardable.is_a?(::Note) + note = award_emoji.awardable + data[:note] = note.hook_attrs + noteable = note.noteable + else + noteable = award_emoji.awardable + end + + if noteable.respond_to?(:hook_attrs) + data[noteable.class.underscore.to_sym] = noteable.hook_attrs + else + Gitlab::AppLogger.error( + "Error building payload data for emoji webhook. #{noteable.class} does not respond to hook_attrs.") + end + + data + end + + def build_base_data(project, user, award_emoji, action) + base_data = { + object_kind: 'emoji', + event_type: action, + user: user.hook_attrs, + project_id: project.id, + project: project.hook_attrs, + object_attributes: award_emoji.hook_attrs + } + + base_data[:object_attributes][:awarded_on_url] = Gitlab::UrlBuilder.build(award_emoji.awardable) + base_data + end + end + end +end diff --git a/lib/gitlab/data_builder/push.rb b/lib/gitlab/data_builder/push.rb index f941c57a6dd..46110937132 100644 --- a/lib/gitlab/data_builder/push.rb +++ b/lib/gitlab/data_builder/push.rb @@ -12,6 +12,7 @@ module Gitlab before: "95790bf891e76fee5e1747ab589903a6a1f80f22", after: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", ref: "refs/heads/master", + ref_protected: true, checkout_sha: "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", message: "Hello World", user_id: 4, @@ -55,6 +56,7 @@ module Gitlab # before: String, # after: String, # ref: String, + # ref_protected: Boolean, # user_id: String, # user_name: String, # user_username: String, @@ -116,6 +118,7 @@ module Gitlab before: oldrev, after: newrev, ref: ref, + ref_protected: project.protected_for?(ref), checkout_sha: checkout_sha(project.repository, newrev, ref), message: message, user_id: user.id, diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index da9ebf4ab0f..fd83f27ef31 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -145,7 +145,7 @@ module Gitlab # Database configured. Returns true even if the database is shared def self.has_config?(database_name) ActiveRecord::Base.configurations - .configs_for(env_name: Rails.env, name: database_name.to_s, include_replicas: true) + .configs_for(env_name: Rails.env, name: database_name.to_s, include_hidden: true) .present? end diff --git a/lib/gitlab/database/async_indexes/migration_helpers.rb b/lib/gitlab/database/async_indexes/migration_helpers.rb index d7128a20a0b..db05635c73d 100644 --- a/lib/gitlab/database/async_indexes/migration_helpers.rb +++ b/lib/gitlab/database/async_indexes/migration_helpers.rb @@ -95,7 +95,7 @@ module Gitlab async_index = Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.find_or_create_by!(name: index_name) do |rec| rec.table_name = table_name - rec.definition = definition + rec.definition = definition.to_s.strip end Gitlab::AppLogger.info( diff --git a/lib/gitlab/database/async_indexes/postgres_async_index.rb b/lib/gitlab/database/async_indexes/postgres_async_index.rb index a3c600a4519..98eb282e43f 100644 --- a/lib/gitlab/database/async_indexes/postgres_async_index.rb +++ b/lib/gitlab/database/async_indexes/postgres_async_index.rb @@ -13,6 +13,8 @@ module Gitlab MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH MAX_DEFINITION_LENGTH = 2048 + before_validation :remove_whitespaces + validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH } validates :table_name, presence: true, length: { maximum: MAX_TABLE_NAME_LENGTH } validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH } @@ -29,6 +31,10 @@ module Gitlab private + def remove_whitespaces + definition.strip! if definition.present? + end + def ensure_correct_schema_and_table_name return unless table_name diff --git a/lib/gitlab/database/ci_builds_partitioning.rb b/lib/gitlab/database/ci_builds_partitioning.rb new file mode 100644 index 00000000000..9f8b19f2d23 --- /dev/null +++ b/lib/gitlab/database/ci_builds_partitioning.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class CiBuildsPartitioning + include AsyncDdlExclusiveLeaseGuard + + ATTEMPTS = 5 + LOCK_TIMEOUT = 10.seconds + LEASE_TIMEOUT = 30.minutes + + FK_NAME = :fk_e20479742e_p + TEMP_FK_NAME = :temp_fk_e20479742e_p + NEXT_PARTITION_ID = 101 + BUILDS_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_builds_101' + ANNOTATION_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_job_annotations_101' + RUNNER_MACHINE_PARTITION_NAME = 'gitlab_partitions_dynamic.ci_runner_machine_builds_101' + + def initialize(logger: Gitlab::AppLogger) + @connection = ::Ci::ApplicationRecord.connection + @timing_configuration = Array.new(ATTEMPTS) { [LOCK_TIMEOUT, 3.minutes] } + @logger = logger + end + + def execute + return unless can_execute? + + try_obtain_lease do + swap_foreign_keys + create_new_ci_builds_partition + create_new_job_annotations_partition + create_new_runner_machine_partition + end + + rescue StandardError => e + log_info("Failed to execute: #{e.message}") + end + + private + + attr_reader :connection, :timing_configuration, :logger + + delegate :quote_table_name, :quote_column_name, to: :connection + + def swap_foreign_keys + if new_foreign_key_exists? + log_info('Foreign key already renamed, nothing to do') + + return + end + + with_lock_retries do + connection.execute drop_old_foreign_key_sql + + rename_constraint :p_ci_builds_metadata, TEMP_FK_NAME, FK_NAME + + each_partition do |partition| + rename_constraint partition.identifier, TEMP_FK_NAME, FK_NAME + end + end + + log_info('Foreign key successfully renamed') + end + + def create_new_ci_builds_partition + if connection.table_exists?(BUILDS_PARTITION_NAME) + log_info('p_ci_builds partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_ci_builds_partition_sql + end + + log_info('Partition for p_ci_builds successfully created') + end + + def create_new_job_annotations_partition + if connection.table_exists?(ANNOTATION_PARTITION_NAME) + log_info('p_ci_job_annotations partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_job_annotations_partition_sql + end + + log_info('Partition for p_ci_job_annotations successfully created') + end + + def create_new_runner_machine_partition + if connection.table_exists?(RUNNER_MACHINE_PARTITION_NAME) + log_info('p_ci_runner_machine_builds partition exists, nothing to do') + return + end + + with_lock_retries do + connection.execute new_runner_machine_partition_sql + end + + log_info('Partition for p_ci_runner_machine_builds successfully created') + end + + def can_execute? + return false if process_disabled? + return false unless Gitlab.com? + + if vacuum_running? + log_info('Autovacuum detected') + + return false + end + + true + end + + def process_disabled? + ::Feature.disabled?(:complete_p_ci_builds_partitioning) + end + + def new_foreign_key_exists? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresForeignKey + .by_constrained_table_name_or_identifier(:p_ci_builds_metadata) + .by_referenced_table_name(:p_ci_builds) + .by_name(FK_NAME) + .exists? + end + end + + def vacuum_running? + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresAutovacuumActivity + .wraparound_prevention + .for_tables(%i[ci_builds ci_builds_metadata]) + .any? + end + end + + def drop_old_foreign_key_sql + <<~SQL.squish + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE ci_builds, p_ci_builds_metadata IN ACCESS EXCLUSIVE MODE; + + ALTER TABLE p_ci_builds_metadata DROP CONSTRAINT #{FK_NAME}; + SQL + end + + def rename_constraint(table_name, old_name, new_name) + connection.execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + + def new_ci_builds_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK ci_pipelines, ci_stages IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_builds IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{BUILDS_PARTITION_NAME} + PARTITION OF p_ci_builds + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def new_job_annotations_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_job_annotations IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{ANNOTATION_PARTITION_NAME} + PARTITION OF p_ci_job_annotations + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def new_runner_machine_partition_sql + <<~SQL + SET LOCAL statement_timeout TO '11s'; + + LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_runner_machine_builds IN ACCESS EXCLUSIVE MODE; + + CREATE TABLE IF NOT EXISTS #{RUNNER_MACHINE_PARTITION_NAME} + PARTITION OF p_ci_runner_machine_builds + FOR VALUES IN (#{NEXT_PARTITION_ID}); + SQL + end + + def with_lock_retries(&block) + Gitlab::Database::WithLockRetries.new( + timing_configuration: timing_configuration, + connection: connection, + logger: logger, + klass: self.class + ).run(raise_on_exhaustion: true, &block) + end + + def each_partition(&block) + Gitlab::Database::SharedModel.using_connection(connection) do + Gitlab::Database::PostgresPartitionedTable.each_partition(:p_ci_builds_metadata, &block) + end + end + + def log_info(message) + logger.info(message: message, class: self.class.to_s) + end + + def connection_db_config + ::Ci::ApplicationRecord.connection_db_config + end + + def lease_timeout + LEASE_TIMEOUT + end + end + end +end diff --git a/lib/gitlab/database/database_connection_info.rb b/lib/gitlab/database/database_connection_info.rb index 57ecbcd64ae..f0cafcf041b 100644 --- a/lib/gitlab/database/database_connection_info.rb +++ b/lib/gitlab/database/database_connection_info.rb @@ -6,6 +6,7 @@ module Gitlab :name, :description, :gitlab_schemas, + :lock_gitlab_schemas, :klass, :fallback_database, :db_dir, @@ -20,6 +21,7 @@ module Gitlab self.name = name.to_sym self.gitlab_schemas = gitlab_schemas.map(&:to_sym) self.klass = klass.constantize + self.lock_gitlab_schemas = (lock_gitlab_schemas || []).map(&:to_sym) self.fallback_database = fallback_database&.to_sym self.db_dir = Rails.root.join(db_dir || 'db') end diff --git a/lib/gitlab/database/each_database.rb b/lib/gitlab/database/each_database.rb index b1af62e4875..e1974aac371 100644 --- a/lib/gitlab/database/each_database.rb +++ b/lib/gitlab/database/each_database.rb @@ -4,7 +4,7 @@ module Gitlab module Database module EachDatabase class << self - def each_database_connection(only: nil, include_shared: true) + def each_connection(only: nil, include_shared: true) selected_names = Array.wrap(only) base_models = select_base_models(selected_names) @@ -18,7 +18,6 @@ module Gitlab end end end - alias_method :each_db_connection, :each_database_connection def each_model_connection(models, only_on: nil, &blk) selected_databases = Array.wrap(only_on).map(&:to_sym) diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 9b58284b389..0bd357b7730 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -23,6 +23,21 @@ module Gitlab tables.map { |table| table_schema!(table) }.to_set end + # Mainly used for test tables + # It maps table names prefixes to gitlab_schemas. + # The order of keys matter. Prefixes that contain other prefixes should come first. + IMPLICIT_GITLAB_SCHEMAS = { + '_test_gitlab_main_clusterwide_' => :gitlab_main_clusterwide, + '_test_gitlab_main_cell_' => :gitlab_main_cell, + '_test_gitlab_main_' => :gitlab_main, + '_test_gitlab_ci_' => :gitlab_ci, + '_test_gitlab_embedding_' => :gitlab_embedding, + '_test_gitlab_geo_' => :gitlab_geo, + '_test_gitlab_pm_' => :gitlab_pm, + '_test_' => :gitlab_shared, + 'pg_' => :gitlab_internal + }.freeze + # rubocop:disable Metrics/CyclomaticComplexity def self.table_schema(name) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` @@ -54,19 +69,11 @@ module Gitlab # All tables from `information_schema.` are marked as `internal` return :gitlab_internal if schema_name == 'information_schema' - return :gitlab_main if table_name.start_with?('_test_gitlab_main_') - - return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') - - return :gitlab_embedding if table_name.start_with?('_test_gitlab_embedding_') - - return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') - - # All tables that start with `_test_` without a following schema are shared and ignored - return :gitlab_shared if table_name.start_with?('_test_') + IMPLICIT_GITLAB_SCHEMAS.each do |prefix, gitlab_schema| + return gitlab_schema if table_name.start_with?(prefix) + end - # All `pg_` tables are marked as `internal` - return :gitlab_internal if table_name.start_with?('pg_') + nil end # rubocop:enable Metrics/CyclomaticComplexity diff --git a/lib/gitlab/database/load_balancing/connection_proxy.rb b/lib/gitlab/database/load_balancing/connection_proxy.rb index 02f14e020c1..2c480eb2cdc 100644 --- a/lib/gitlab/database/load_balancing/connection_proxy.rb +++ b/lib/gitlab/database/load_balancing/connection_proxy.rb @@ -39,7 +39,7 @@ module Gitlab @load_balancer = load_balancer end - def select_all(arel, name = nil, binds = [], preparable: nil) + def select_all(arel, name = nil, binds = [], preparable: nil, async: false) if arel.respond_to?(:locked) && arel.locked # SELECT ... FOR UPDATE queries should be sent to the primary. current_session.write! diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 23476e1f5e9..f6144b7b772 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -12,6 +12,8 @@ module Gitlab REPLICA_SUFFIX = '_replica' + attr_accessor :service_discovery + attr_reader :host_list, :configuration # configuration - An instance of `LoadBalancing::Configuration` that @@ -45,6 +47,8 @@ module Gitlab # If no secondaries were available this method will use the primary # instead. def read(&block) + service_discovery&.log_refresh_thread_interruption + conflict_retried = 0 while host @@ -103,6 +107,8 @@ module Gitlab # Yields a connection that can be used for both reads and writes. def read_write + service_discovery&.log_refresh_thread_interruption + connection = nil transaction_open = nil @@ -285,7 +291,7 @@ module Gitlab def pool ActiveRecord::Base.connection_handler.retrieve_connection_pool( @configuration.connection_specification_name, - role: ActiveRecord::Base.writing_role, + role: ActiveRecord.writing_role, shard: ActiveRecord::Base.default_shard ) || raise(::ActiveRecord::ConnectionNotEstablished) end diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index 57a588db8a8..a0b0ad19f73 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -15,12 +15,14 @@ module Gitlab class ServiceDiscovery EmptyDnsResponse = Class.new(StandardError) + attr_accessor :refresh_thread, :refresh_thread_last_run, :refresh_thread_interruption_logged + attr_reader :interval, :record, :record_type, :disconnect_timeout, :load_balancer MAX_SLEEP_ADJUSTMENT = 10 - MAX_DISCOVERY_RETRIES = 3 + DISCOVERY_THREAD_REFRESH_DELTA = 5 RETRY_DELAY_RANGE = (0.1..0.2).freeze @@ -74,8 +76,10 @@ module Gitlab # rubocop:enable Metrics/ParameterLists def start - Thread.new do + self.refresh_thread = Thread.new do loop do + self.refresh_thread_last_run = Time.current + next_sleep_duration = perform_service_discovery # We slightly randomize the sleep() interval. This should reduce @@ -103,15 +107,6 @@ module Gitlab # Slightly randomize the retry delay so that, in the case of a total # dns outage, all starting services do not pressure the dns server at the same time. sleep(rand(RETRY_DELAY_RANGE)) - rescue Exception => error # rubocop:disable Lint/RescueException - # All exceptions are logged to find any pattern and solve https://gitlab.com/gitlab-org/gitlab/-/issues/364370 - # This will be removed in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/120173 - Gitlab::Database::LoadBalancing::Logger.error( - event: :service_discovery_unexpected_exception, - message: "Service discovery encountered an uncaught error: #{error.message}" - ) - - raise end interval @@ -214,6 +209,20 @@ module Gitlab ) end + def log_refresh_thread_interruption + return if refresh_thread_last_run.blank? || refresh_thread_interruption_logged || + (refresh_thread_last_run + DISCOVERY_THREAD_REFRESH_DELTA.minutes).future? + + Gitlab::Database::LoadBalancing::Logger.error( + event: :service_discovery_refresh_thread_interrupt, + refresh_thread_last_run: refresh_thread_last_run, + thread_status: refresh_thread&.status&.to_s, + thread_backtrace: refresh_thread&.backtrace&.join('\n') + ) + + self.refresh_thread_interruption_logged = true + end + private def record_type_for(type) diff --git a/lib/gitlab/database/load_balancing/setup.rb b/lib/gitlab/database/load_balancing/setup.rb index eceea1d8d9c..2e65e1c8e56 100644 --- a/lib/gitlab/database/load_balancing/setup.rb +++ b/lib/gitlab/database/load_balancing/setup.rb @@ -55,6 +55,8 @@ module Gitlab sv = ServiceDiscovery.new(load_balancer, **configuration.service_discovery) + load_balancer.service_discovery = sv + sv.perform_service_discovery sv.start if @start_service_discovery diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index 291f483e6e4..256c524e989 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -11,6 +11,7 @@ module Gitlab include Migrations::ConstraintsHelpers include Migrations::ExtensionHelpers include Migrations::SidekiqHelpers + include Migrations::RedisHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers diff --git a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb index 55c4fd6a7af..fe456fab505 100644 --- a/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb +++ b/lib/gitlab/database/migration_helpers/automatic_lock_writes_on_tables.rb @@ -11,7 +11,9 @@ module Gitlab end def exec_migration(connection, direction) - return super if %w[main ci].exclude?(Gitlab::Database.db_config_name(connection)) + db_config_name = Gitlab::Database.db_config_name(connection) + db_info = Gitlab::Database.all_database_connections.fetch(db_config_name) + return super if db_info.lock_gitlab_schemas.empty? return super if automatic_lock_on_writes_disabled? # This compares the tables only on the `public` schema. Partitions are not affected @@ -20,7 +22,7 @@ module Gitlab new_tables = connection.tables - tables new_tables.each do |table_name| - lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(table_name) + lock_writes_on_table(connection, table_name) if should_lock_writes_on_table?(db_info, table_name) end end @@ -39,16 +41,17 @@ module Gitlab end end - def should_lock_writes_on_table?(table_name) - # currently gitlab_schema represents only present existing tables, this is workaround for deleted tables - # that should be skipped as they will be removed in a future migration. + def should_lock_writes_on_table?(db_info, table_name) + # We skip locking writes on tables that are scheduled for deletion in a future migration return false if Gitlab::Database::GitlabSchema.deleted_tables_to_schema[table_name] table_schema = Gitlab::Database::GitlabSchema.table_schema!(table_name.to_s) - return false unless %i[gitlab_main gitlab_ci].include?(table_schema) - - Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) + # This takes into consideration which database mode is used. + # In single-db and single-db-ci-connection the main database includes gitlab_ci tables, + # so we don't lock them there. + Gitlab::Database.gitlab_schemas_for_connection(connection).exclude?(table_schema) && + db_info.lock_gitlab_schemas.include?(table_schema) end # with_retries creates new a transaction. So we set it to false if the connection is diff --git a/lib/gitlab/database/migrations/redis_helpers.rb b/lib/gitlab/database/migrations/redis_helpers.rb new file mode 100644 index 00000000000..41a2841da7c --- /dev/null +++ b/lib/gitlab/database/migrations/redis_helpers.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module RedisHelpers + SCAN_START_CURSOR = '0' + + # Check if the migration exists before enqueueing the worker + def queue_redis_migration_job(job_name) + RedisMigrationWorker.fetch_migrator!(job_name) + RedisMigrationWorker.perform_async(job_name, SCAN_START_CURSOR) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index ed55081c9ab..dc9ea304aac 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -32,7 +32,7 @@ module Gitlab result_dir = background_migrations_dir(for_database, legacy_mode) # Only one loop iteration since we pass `only:` here - Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + Gitlab::Database::EachDatabase.each_connection(only: for_database) do |connection| from_id = batched_migrations_last_id(for_database).read runner = Gitlab::Database::Migrations::TestBatchedBackgroundRunner @@ -68,7 +68,7 @@ module Gitlab runner = nil base_dir = background_migrations_dir(for_database, false) - Gitlab::Database::EachDatabase.each_database_connection(only: for_database) do |connection| + Gitlab::Database::EachDatabase.each_connection(only: for_database) do |connection| runner = Gitlab::Database::Migrations::BatchedMigrationLastId .new(connection, base_dir) end diff --git a/lib/gitlab/database/migrations/test_batched_background_runner.rb b/lib/gitlab/database/migrations/test_batched_background_runner.rb index af853c933ba..c5e0b361df5 100644 --- a/lib/gitlab/database/migrations/test_batched_background_runner.rb +++ b/lib/gitlab/database/migrations/test_batched_background_runner.rb @@ -57,6 +57,20 @@ module Gitlab job_arguments: migration.job_arguments ) + # If no rows match, the next_bounds are nil. + # This will only happen if there are zero rows to match from the current sampling point to the end + # of the table + # Simulate the approach in the actual background migration worker by not sampling a batch + # from this range. + # (The actual worker would finish the migration, but we may find batches that can be sampled elsewhere + # in the table) + if next_bounds.nil? + # If the migration has no work to do across the entire table, sampling can get stuck + # in a loop if we don't mark the attempted batches as completed + completed_batches << (batch_start..(batch_start + migration.batch_size)) + next + end + batch_min, batch_max = next_bounds job = migration.create_batched_job!(batch_min, batch_max) @@ -65,7 +79,7 @@ module Gitlab job end - end + end.reject(&:nil?) # Remove skipped batches from the lazy list of batches to test job_class_name = migration.job_class_name diff --git a/lib/gitlab/database/partitioning.rb b/lib/gitlab/database/partitioning.rb index 9895a68ec8d..48f58920d52 100644 --- a/lib/gitlab/database/partitioning.rb +++ b/lib/gitlab/database/partitioning.rb @@ -40,7 +40,7 @@ module Gitlab next if model < ::Gitlab::Database::SharedModel && !(model < TableWithoutModel) model_connection_name = model.connection_db_config.name - Gitlab::Database::EachDatabase.each_db_connection(include_shared: false) do |connection, connection_name| + Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, connection_name| if connection_name != model_connection_name PartitionManager.new(model, connection: connection).sync_partitions end @@ -64,7 +64,7 @@ module Gitlab Gitlab::AppLogger.info(message: 'Dropping detached postgres partitions') - Gitlab::Database::EachDatabase.each_database_connection do + Gitlab::Database::EachDatabase.each_connection do DetachedPartitionDropper.new.perform end diff --git a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb b/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb deleted file mode 100644 index 88affaa9757..00000000000 --- a/lib/gitlab/database/postgresql_adapter/empty_query_ping.rb +++ /dev/null @@ -1,25 +0,0 @@ -# frozen_string_literal: true - -# This patch will be included in the next Rails release: https://github.com/rails/rails/pull/42368 -raise 'This patch can be removed' if Rails::VERSION::MAJOR > 6 - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Database - module PostgresqlAdapter - module EmptyQueryPing - # ActiveRecord uses `SELECT 1` to check if the connection is alive - # We patch this here to use an empty query instead, which is a bit faster - def active? - @lock.synchronize do - @connection.query ';' - end - true - rescue PG::Error - false - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/database/reindexing.rb b/lib/gitlab/database/reindexing.rb index 739e573b6c4..9c860ebc6aa 100644 --- a/lib/gitlab/database/reindexing.rb +++ b/lib/gitlab/database/reindexing.rb @@ -20,7 +20,7 @@ module Gitlab end def self.invoke(database = nil) - Gitlab::Database::EachDatabase.each_database_connection do |connection, connection_name| + Gitlab::Database::EachDatabase.each_connection do |connection, connection_name| next if database && database.to_s != connection_name.to_s Gitlab::Database::SharedModel.logger = Logger.new($stdout) if Gitlab::Utils.to_boolean(ENV['LOG_QUERIES_TO_CONSOLE'], default: false) @@ -59,6 +59,7 @@ module Gitlab # most bloated indexes for reindexing. def self.perform_with_heuristic(candidate_indexes = Gitlab::Database::PostgresIndex.reindexing_support, maximum_records: DEFAULT_INDEXES_PER_INVOCATION) IndexSelection.new(candidate_indexes).take(maximum_records).each do |index| + Gitlab::Database::CiBuildsPartitioning.new.execute Coordinator.new(index).perform end end diff --git a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb deleted file mode 100644 index 32d638380ea..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/column_database_adapter.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ColumnDatabaseAdapter - def initialize(query_result) - @query_result = query_result - end - - def name - @name ||= query_result['column_name'] - end - - def table_name - query_result['table_name'] - end - - def data_type - query_result['data_type'] - end - - def default - return unless query_result['column_default'] - - return if name == 'id' || query_result['column_default'].include?('nextval') - - "DEFAULT #{query_result['column_default']}" - end - - def nullable - 'NOT NULL' if query_result['not_null'] - end - - def partition_key? - query_result['partition_key'] - end - - private - - attr_reader :query_result - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb deleted file mode 100644 index 20814b098c1..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/column_structure_sql_adapter.rb +++ /dev/null @@ -1,139 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - UndefinedPGType = Class.new(StandardError) - - class ColumnStructureSqlAdapter - NOT_NULL_CONSTR = :CONSTR_NOTNULL - DEFAULT_CONSTR = :CONSTR_DEFAULT - - MAPPINGS = { - 't' => 'true', - 'f' => 'false' - }.freeze - - attr_reader :table_name - - def initialize(table_name, pg_query_stmt, partitioning_stmt) - @table_name = table_name - @pg_query_stmt = pg_query_stmt - @partitioning_stmt = partitioning_stmt - end - - def name - @name ||= pg_query_stmt.colname - end - - def data_type - type(pg_query_stmt.type_name) - end - - def default - return if name == 'id' - - value = parse_node(constraints.find { |node| node.constraint.contype == DEFAULT_CONSTR }) - - return unless value - - "DEFAULT #{value}" - end - - def nullable - 'NOT NULL' if constraints.any? { |node| node.constraint.contype == NOT_NULL_CONSTR } - end - - def partition_key? - partition_keys.include?(name) - end - - private - - attr_reader :pg_query_stmt, :partitioning_stmt - - def constraints - @constraints ||= pg_query_stmt.constraints - end - - # Returns the node type - # - # pg_type:: type alias, used internally by postgres, +int4+, +int8+, +bool+, +varchar+ - # type:: type name, like +integer+, +bigint+, +boolean+, +character varying+. - # array_ext:: adds the +[]+ extension for array types. - # precision_ext:: adds the precision, if have any, like +(255)+, +(6)+. - # - # @info +timestamp+ and +timestamptz+ have a particular case when precision is defined. - # In this case, the order of the statement needs to be re-arranged from - # timestamp without time zone(6) to timestamp(6) without a time zone. - def type(node) - pg_type = parse_node(node.names.last) - type = PgTypes::TYPES.fetch(pg_type).dup - array_ext = '[]' if node.array_bounds.any? - precision_ext = "(#{node.typmods.map { |typmod| parse_node(typmod) }.join(',')})" if node.typmods.any? - - if %w[timestamp timestamptz].include?(pg_type) - type.gsub!('timestamp', ['timestamp', precision_ext].compact.join('')) - precision_ext = nil - end - - [type, precision_ext, array_ext].compact.join('') - rescue KeyError => exception - raise UndefinedPGType, exception.message - end - - # Parses PGQuery nodes recursively - # - # :constraint:: nodes that groups column default info - # :partition_elem:: node that store partition key info - # :func_cal:: nodes that stores functions, like +now()+ - # :a_const:: nodes that stores constant values, like +t+, +f+, +0.0.0.0+, +255+, +1.0+ - # :type_cast:: nodes that stores casting values, like +'name'::text+, +'0.0.0.0'::inet+ - # else:: extract node values in the last iteration of the recursion, like +int4+, +1.0+, +now+, +255+ - # - # @note boolean types types are mapped from +t+, +f+ to +true+, +false+ - def parse_node(node) - return unless node - - case node.node - when :constraint - parse_node(node.constraint.raw_expr) - when :partition_elem - node.partition_elem.name - when :func_call - "#{parse_node(node.func_call.funcname.first)}()" - when :a_const - parse_a_const(node.a_const) - when :type_cast - value = parse_node(node.type_cast.arg) - type = type(node.type_cast.type_name) - separator = MAPPINGS.key?(value) ? '' : "::#{type}" - - [MAPPINGS.fetch(value, "'#{value}'"), separator].compact.join('') - else - get_value_from_key(node, key: node.node) - end - end - - def parse_a_const(a_const) - return unless a_const - - type = a_const.val - get_value_from_key(a_const, key: type) - end - - def get_value_from_key(node, key:) - node.to_h[key].values.last - end - - def partition_keys - return [] unless partitioning_stmt - - @partition_keys ||= partitioning_stmt.part_params.map { |key_stmt| parse_node(key_stmt) } - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb deleted file mode 100644 index 3b45f5c77ca..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/foreign_key_database_adapter.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ForeignKeyDatabaseAdapter - def initialize(query_result) - @query_result = query_result - end - - def name - "#{query_result['schema']}.#{query_result['foreign_key_name']}" - end - - def table_name - query_result['table_name'] - end - - def statement - query_result['foreign_key_definition'] - end - - private - - attr_reader :query_result - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb b/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb deleted file mode 100644 index e4c1e1adab3..00000000000 --- a/lib/gitlab/database/schema_validation/adapters/foreign_key_structure_sql_adapter.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Adapters - class ForeignKeyStructureSqlAdapter - STATEMENT_REGEX = /\bREFERENCES\s\K\S+\K\s\(/ - EXTRACT_REGEX = /\bFOREIGN KEY.*/ - - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - "#{schema_name}.#{foreign_key_name}" - end - - def table_name - parsed_stmt.relation.relname - end - - # PgQuery parses FK statements with an extra space in the referenced table column. - # This extra space needs to be removed. - # - # @example REFERENCES ci_pipelines (id) => REFERENCES ci_pipelines(id) - def statement - deparse_stmt[EXTRACT_REGEX].gsub(STATEMENT_REGEX, '(') - end - - private - - attr_reader :parsed_stmt - - def schema_name - parsed_stmt.relation.schemaname - end - - def foreign_key_name - parsed_stmt.cmds.first.alter_table_cmd.def.constraint.conname - end - - def deparse_stmt - PgQuery.deparse_stmt(parsed_stmt) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/database.rb b/lib/gitlab/database/schema_validation/database.rb deleted file mode 100644 index 858bf618f44..00000000000 --- a/lib/gitlab/database/schema_validation/database.rb +++ /dev/null @@ -1,166 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Database - STATIC_PARTITIONS_SCHEMA = 'gitlab_partitions_static' - - def initialize(connection) - @connection = connection - end - - def fetch_index_by_name(index_name) - index_map[index_name] - end - - def fetch_trigger_by_name(trigger_name) - trigger_map[trigger_name] - end - - def fetch_foreign_key_by_name(foreign_key_name) - foreign_key_map[foreign_key_name] - end - - def fetch_table_by_name(table_name) - table_map[table_name] - end - - def index_exists?(index_name) - index_map[index_name].present? - end - - def trigger_exists?(trigger_name) - trigger_map[trigger_name].present? - end - - def foreign_key_exists?(foreign_key_name) - fetch_foreign_key_by_name(foreign_key_name).present? - end - - def table_exists?(table_name) - fetch_table_by_name(table_name).present? - end - - def indexes - index_map.values - end - - def triggers - trigger_map.values - end - - def foreign_keys - foreign_key_map.values - end - - def tables - table_map.values - end - - private - - attr_reader :connection - - def schemas - @schemas ||= [STATIC_PARTITIONS_SCHEMA, connection.current_schema] - end - - def index_map - @index_map ||= - fetch_indexes.transform_values! do |index_stmt| - SchemaObjects::Index.new(PgQuery.parse(index_stmt).tree.stmts.first.stmt.index_stmt) - end - end - - def trigger_map - @trigger_map ||= - fetch_triggers.transform_values! do |trigger_stmt| - SchemaObjects::Trigger.new(PgQuery.parse(trigger_stmt).tree.stmts.first.stmt.create_trig_stmt) - end - end - - def foreign_key_map - @foreign_key_map ||= fetch_fks.each_with_object({}) do |stmt, result| - adapter = Adapters::ForeignKeyDatabaseAdapter.new(stmt) - - result[adapter.name] = SchemaObjects::ForeignKey.new(adapter) - end - end - - def table_map - @table_map ||= fetch_tables.transform_values! do |stmt| - columns = stmt.map { |column| SchemaObjects::Column.new(Adapters::ColumnDatabaseAdapter.new(column)) } - - SchemaObjects::Table.new(stmt.first['table_name'], columns) - end - end - - def fetch_indexes - sql = <<~SQL - SELECT indexname, indexdef - FROM pg_indexes - WHERE indexname NOT LIKE '%_pkey' AND schemaname IN ($1, $2); - SQL - - connection.select_rows(sql, nil, schemas).to_h - end - - def fetch_triggers - sql = <<~SQL - SELECT triggers.tgname, pg_get_triggerdef(triggers.oid) - FROM pg_catalog.pg_trigger triggers - INNER JOIN pg_catalog.pg_class rel ON triggers.tgrelid = rel.oid - INNER JOIN pg_catalog.pg_namespace nsp ON nsp.oid = rel.relnamespace - WHERE triggers.tgisinternal IS FALSE - AND nsp.nspname IN ($1, $2) - SQL - - connection.select_rows(sql, nil, schemas).to_h - end - - def fetch_tables - sql = <<~SQL - SELECT - table_information.relname AS table_name, - col_information.attname AS column_name, - col_information.attnotnull AS not_null, - col_information.attnum = ANY(pg_partitioned_table.partattrs) as partition_key, - format_type(col_information.atttypid, col_information.atttypmod) AS data_type, - pg_get_expr(col_default_information.adbin, col_default_information.adrelid) AS column_default - FROM pg_attribute AS col_information - JOIN pg_class AS table_information ON col_information.attrelid = table_information.oid - JOIN pg_namespace AS schema_information ON table_information.relnamespace = schema_information.oid - LEFT JOIN pg_partitioned_table ON pg_partitioned_table.partrelid = table_information.oid - LEFT JOIN pg_attrdef AS col_default_information ON col_information.attrelid = col_default_information.adrelid - AND col_information.attnum = col_default_information.adnum - WHERE NOT col_information.attisdropped - AND col_information.attnum > 0 - AND table_information.relkind IN ('r', 'p') - AND schema_information.nspname IN ($1, $2) - SQL - - connection.exec_query(sql, nil, schemas).group_by { |row| row['table_name'] } - end - - def fetch_fks - sql = <<~SQL - SELECT - pg_namespace.nspname::text AS schema, - pg_class.relname::text AS table_name, - pg_constraint.conname AS foreign_key_name, - pg_get_constraintdef(pg_constraint.oid) AS foreign_key_definition - FROM pg_constraint - INNER JOIN pg_class ON pg_constraint.conrelid = pg_class.oid - INNER JOIN pg_namespace ON pg_class.relnamespace = pg_namespace.oid - WHERE contype = 'f' - AND pg_namespace.nspname = $1 - AND pg_constraint.conparentid = 0 - SQL - - connection.exec_query(sql, nil, [connection.current_schema]) - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/inconsistency.rb b/lib/gitlab/database/schema_validation/inconsistency.rb deleted file mode 100644 index 766f48ef339..00000000000 --- a/lib/gitlab/database/schema_validation/inconsistency.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Inconsistency - def initialize(validator_class, structure_sql_object, database_object) - @validator_class = validator_class - @structure_sql_object = structure_sql_object - @database_object = database_object - end - - def error_message - format(validator_class::ERROR_MESSAGE, object_name) - end - - def type - validator_class.name.demodulize.underscore - end - - def object_type - structure_sql_object&.class&.name&.demodulize || database_object&.class&.name&.demodulize - end - - def table_name - structure_sql_object&.table_name || database_object&.table_name - end - - def object_name - structure_sql_object&.name || database_object&.name - end - - def diff - Diffy::Diff.new(structure_sql_statement, database_statement) - end - - def inspect - <<~MSG - #{'-' * 54} - #{error_message} - Diff: - #{diff.to_s(:color)} - #{'-' * 54} - MSG - end - - def structure_sql_statement - return unless structure_sql_object - - "#{structure_sql_object.statement}\n" - end - - def database_statement - return unless database_object - - "#{database_object.statement}\n" - end - - private - - attr_reader :validator_class, :structure_sql_object, :database_object - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/pg_types.rb b/lib/gitlab/database/schema_validation/pg_types.rb deleted file mode 100644 index 0a1999d056e..00000000000 --- a/lib/gitlab/database/schema_validation/pg_types.rb +++ /dev/null @@ -1,73 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class PgTypes - TYPES = { - 'bool' => 'boolean', - 'bytea' => 'bytea', - 'char' => '"char"', - 'int8' => 'bigint', - 'int2' => 'smallint', - 'int4' => 'integer', - 'regproc' => 'regproc', - 'text' => 'text', - 'oid' => 'oid', - 'tid' => 'tid', - 'xid' => 'xid', - 'cid' => 'cid', - 'json' => 'json', - 'xml' => 'xml', - 'pg_node_tree' => 'pg_node_tree', - 'pg_ndistinct' => 'pg_ndistinct', - 'pg_dependencies' => 'pg_dependencies', - 'pg_mcv_list' => 'pg_mcv_list', - 'xid8' => 'xid8', - 'path' => 'path', - 'polygon' => 'polygon', - 'float4' => 'real', - 'float8' => 'double precision', - 'circle' => 'circle', - 'money' => 'money', - 'macaddr' => 'macaddr', - 'inet' => 'inet', - 'cidr' => 'cidr', - 'macaddr8' => 'macaddr8', - 'aclitem' => 'aclitem', - 'bpchar' => 'character', - 'varchar' => 'character varying', - 'date' => 'date', - 'time' => 'time without time zone', - 'timestamp' => 'timestamp without time zone', - 'timestamptz' => 'timestamp with time zone', - 'interval' => 'interval', - 'timetz' => 'time with time zone', - 'bit' => 'bit', - 'varbit' => 'bit varying', - 'numeric' => 'numeric', - 'refcursor' => 'refcursor', - 'regprocedure' => 'regprocedure', - 'regoper' => 'regoper', - 'regoperator' => 'regoperator', - 'regclass' => 'regclass', - 'regcollation' => 'regcollation', - 'regtype' => 'regtype', - 'regrole' => 'regrole', - 'regnamespace' => 'regnamespace', - 'uuid' => 'uuid', - 'pg_lsn' => 'pg_lsn', - 'tsvector' => 'tsvector', - 'gtsvector' => 'gtsvector', - 'tsquery' => 'tsquery', - 'regconfig' => 'regconfig', - 'regdictionary' => 'regdictionary', - 'jsonb' => 'jsonb', - 'jsonpath' => 'jsonpath', - 'txid_snapshot' => 'txid_snapshot', - 'pg_snapshot' => 'pg_snapshot' - }.freeze - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/runner.rb b/lib/gitlab/database/schema_validation/runner.rb deleted file mode 100644 index 7a02c8a16d6..00000000000 --- a/lib/gitlab/database/schema_validation/runner.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class Runner - def initialize(structure_sql, database, validators: Validators::BaseValidator.all_validators) - @structure_sql = structure_sql - @database = database - @validators = validators - end - - def execute - validators.flat_map { |c| c.new(structure_sql, database).execute } - end - - private - - attr_reader :structure_sql, :database, :validators - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/base.rb b/lib/gitlab/database/schema_validation/schema_objects/base.rb deleted file mode 100644 index 43d30dc54ae..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/base.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Base - def initialize(parsed_stmt) - @parsed_stmt = parsed_stmt - end - - def name - raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" - end - - def table_name - parsed_stmt.relation.relname - end - - def statement - @statement ||= PgQuery.deparse_stmt(parsed_stmt) - end - - private - - attr_reader :parsed_stmt - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/column.rb b/lib/gitlab/database/schema_validation/schema_objects/column.rb deleted file mode 100644 index bd219300a13..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/column.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Column - def initialize(adapter) - @adapter = adapter - end - - attr_reader :adapter - - delegate :name, :table_name, :partition_key?, to: :adapter - - def statement - [name, adapter.data_type, adapter.default, adapter.nullable].compact.join(' ') - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb b/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb deleted file mode 100644 index b616b1a72b7..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/foreign_key.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class ForeignKey - def initialize(adapter) - @adapter = adapter - end - - # Foreign key name should include the schema, as the same name could be used across different schemas - # - # @example public.foreign_key_name - def name - @name ||= adapter.name - end - - def table_name - @table_name ||= adapter.table_name - end - - def statement - @statement ||= adapter.statement - end - - private - - attr_reader :adapter - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/index.rb b/lib/gitlab/database/schema_validation/schema_objects/index.rb deleted file mode 100644 index 28d61b18266..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/index.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Index < Base - def name - parsed_stmt.idxname - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/table.rb b/lib/gitlab/database/schema_validation/schema_objects/table.rb deleted file mode 100644 index de2e675d20e..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/table.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Table - def initialize(name, columns) - @name = name - @columns = columns - end - - attr_reader :name, :columns - - def table_name - name - end - - def statement - format('CREATE TABLE %s (%s)', name, columns_statement) - end - - def fetch_column_by_name(column_name) - columns.find { |column| column.name == column_name } - end - - def column_exists?(column_name) - fetch_column_by_name(column_name).present? - end - - private - - def columns_statement - columns.reject(&:partition_key?).map(&:statement).join(', ') - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb b/lib/gitlab/database/schema_validation/schema_objects/trigger.rb deleted file mode 100644 index 508e6b27ed3..00000000000 --- a/lib/gitlab/database/schema_validation/schema_objects/trigger.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module SchemaObjects - class Trigger < Base - def name - parsed_stmt.trigname - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/structure_sql.rb b/lib/gitlab/database/schema_validation/structure_sql.rb deleted file mode 100644 index 4d6fa17f0fc..00000000000 --- a/lib/gitlab/database/schema_validation/structure_sql.rb +++ /dev/null @@ -1,125 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class StructureSql - DEFAULT_SCHEMA = 'public' - - def initialize(structure_file_path, schema_name = DEFAULT_SCHEMA) - @structure_file_path = structure_file_path - @schema_name = schema_name - end - - def index_exists?(index_name) - indexes.find { |index| index.name == index_name }.present? - end - - def trigger_exists?(trigger_name) - triggers.find { |trigger| trigger.name == trigger_name }.present? - end - - def foreign_key_exists?(foreign_key_name) - foreign_keys.find { |fk| fk.name == foreign_key_name }.present? - end - - def fetch_table_by_name(table_name) - tables.find { |table| table.name == table_name } - end - - def table_exists?(table_name) - fetch_table_by_name(table_name).present? - end - - def indexes - @indexes ||= map_with_default_schema(index_statements, SchemaObjects::Index) - end - - def triggers - @triggers ||= map_with_default_schema(trigger_statements, SchemaObjects::Trigger) - end - - def foreign_keys - @foreign_keys ||= foreign_key_statements.map do |stmt| - stmt.relation.schemaname = schema_name if stmt.relation.schemaname == '' - - SchemaObjects::ForeignKey.new(Adapters::ForeignKeyStructureSqlAdapter.new(stmt)) - end - end - - def tables - @tables ||= table_statements.map do |stmt| - table_name = stmt.relation.relname - partition_stmt = stmt.partspec - - columns = stmt.table_elts.select { |n| n.node == :column_def }.map do |column| - adapter = Adapters::ColumnStructureSqlAdapter.new(table_name, column.column_def, partition_stmt) - SchemaObjects::Column.new(adapter) - end - - SchemaObjects::Table.new(table_name, columns) - end - end - - private - - attr_reader :structure_file_path, :schema_name - - def index_statements - statements.filter_map { |s| s.stmt.index_stmt } - end - - def trigger_statements - statements.filter_map { |s| s.stmt.create_trig_stmt } - end - - def table_statements - statements.filter_map { |s| s.stmt.create_stmt } - end - - def foreign_key_statements - constraint_statements(:CONSTR_FOREIGN) - end - - # Filter constraint statement nodes - # - # @param constraint_type [Symbol] node type. One of CONSTR_PRIMARY, CONSTR_CHECK, CONSTR_EXCLUSION, - # CONSTR_UNIQUE or CONSTR_FOREIGN. - def constraint_statements(constraint_type) - alter_table_statements(:AT_AddConstraint).filter do |stmt| - stmt.cmds.first.alter_table_cmd.def.constraint.contype == constraint_type - end - end - - # Filter alter table statement nodes - # - # @param subtype [Symbol] node subtype +AT_AttachPartition+, +AT_ColumnDefault+ or +AT_AddConstraint+ - def alter_table_statements(subtype) - statements.filter_map do |statement| - node = statement.stmt.alter_table_stmt - - next unless node - - node if node.cmds.first.alter_table_cmd.subtype == subtype - end - end - - def statements - @statements ||= parsed_structure_file.tree.stmts - end - - def parsed_structure_file - PgQuery.parse(File.read(structure_file_path)) - end - - def map_with_default_schema(statements, validation_class) - statements.map do |statement| - statement.relation.schemaname = schema_name if statement.relation.schemaname == '' - - validation_class.new(statement) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/track_inconsistency.rb b/lib/gitlab/database/schema_validation/track_inconsistency.rb deleted file mode 100644 index 6e167653d32..00000000000 --- a/lib/gitlab/database/schema_validation/track_inconsistency.rb +++ /dev/null @@ -1,138 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - class TrackInconsistency - COLUMN_TEXT_LIMIT = 6144 - - def initialize(inconsistency, project, user) - @inconsistency = inconsistency - @project = project - @user = user - end - - def execute - return unless Gitlab.com? - return refresh_issue if inconsistency_record.present? - - result = ::Issues::CreateService.new( - container: project, - current_user: user, - params: params, - perform_spam_check: false).execute - - track_inconsistency(result[:issue]) if result.success? - end - - private - - attr_reader :inconsistency, :project, :user - - def track_inconsistency(issue) - schema_inconsistency_model.create!( - issue: issue, - object_name: inconsistency.object_name, - table_name: inconsistency.table_name, - valitador_name: inconsistency.type, - diff: inconsistency_diff - ) - end - - def params - { - title: issue_title, - description: description, - issue_type: 'issue', - labels: default_labels + group_labels - } - end - - def issue_title - "New schema inconsistency: #{inconsistency.object_name}" - end - - def description - <<~MSG - We have detected a new schema inconsistency. - - **Table name:** #{inconsistency.table_name}\ - **Object name:** #{inconsistency.object_name}\ - **Validator name:** #{inconsistency.type}\ - **Object type:** #{inconsistency.object_type}\ - **Error message:** #{inconsistency.error_message} - - - **Structure.sql statement:** - - ```sql - #{inconsistency.structure_sql_statement} - ``` - - **Database statement:** - - ```sql - #{inconsistency.database_statement} - ``` - - **Diff:** - - ```diff - #{inconsistency.diff} - - ``` - - - For more information, please contact the database team. - MSG - end - - def group_labels - dictionary = YAML.safe_load(File.read(table_file_path)) - - dictionary['feature_categories'].to_a.filter_map do |feature_category| - Gitlab::Database::ConvertFeatureCategoryToGroupLabel.new(feature_category).execute - end - rescue Errno::ENOENT - [] - end - - def default_labels - %w[database database-inconsistency-report type::maintenance severity::4] - end - - def table_file_path - Rails.root.join(Gitlab::Database::GitlabSchema.dictionary_paths.first, "#{inconsistency.table_name}.yml") - end - - def schema_inconsistency_model - Gitlab::Database::SchemaValidation::SchemaInconsistency - end - - def refresh_issue - return if inconsistency_diff == inconsistency_record.diff # Nothing to refresh - - note = ::Notes::CreateService.new( - inconsistency_record.issue.project, - user, - { noteable_type: 'Issue', noteable: inconsistency_record.issue, note: description } - ).execute - - inconsistency_record.update!(diff: inconsistency_diff) if note.persisted? - end - - def inconsistency_diff - @inconsistency_diff ||= inconsistency.diff.to_s.first(COLUMN_TEXT_LIMIT) - end - - def inconsistency_record - @inconsistency_record ||= schema_inconsistency_model.with_open_issues.find_by( - object_name: inconsistency.object_name, - table_name: inconsistency.table_name, - valitador_name: inconsistency.type - ) - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/base_validator.rb b/lib/gitlab/database/schema_validation/validators/base_validator.rb deleted file mode 100644 index ee322e50a2c..00000000000 --- a/lib/gitlab/database/schema_validation/validators/base_validator.rb +++ /dev/null @@ -1,49 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class BaseValidator - ERROR_MESSAGE = 'A schema inconsistency has been found' - - def initialize(structure_sql, database) - @structure_sql = structure_sql - @database = database - end - - def self.all_validators - [ - ExtraTables, - ExtraTableColumns, - ExtraIndexes, - ExtraTriggers, - ExtraForeignKeys, - MissingTables, - MissingTableColumns, - MissingIndexes, - MissingTriggers, - MissingForeignKeys, - DifferentDefinitionTables, - DifferentDefinitionIndexes, - DifferentDefinitionTriggers, - DifferentDefinitionForeignKeys - ] - end - - def execute - raise NoMethodError, "subclasses of #{self.class.name} must implement #{__method__}" - end - - private - - attr_reader :structure_sql, :database - - def build_inconsistency(validator_class, structure_sql_object, database_object) - Inconsistency.new(validator_class, structure_sql_object, database_object) - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb deleted file mode 100644 index 8969fa76cd8..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_foreign_keys.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionForeignKeys < BaseValidator - ERROR_MESSAGE = "The %s foreign key has a different statement between structure.sql and database" - - def execute - structure_sql.foreign_keys.filter_map do |structure_sql_fk| - database_fk = database.fetch_foreign_key_by_name(structure_sql_fk.name) - - next if database_fk.nil? - next if database_fk.statement == structure_sql_fk.statement - - build_inconsistency(self.class, structure_sql_fk, database_fk) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb b/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb deleted file mode 100644 index ba12b3cdc61..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_indexes.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionIndexes < BaseValidator - ERROR_MESSAGE = "The %s index has a different statement between structure.sql and database" - - def execute - structure_sql.indexes.filter_map do |structure_sql_index| - database_index = database.fetch_index_by_name(structure_sql_index.name) - - next if database_index.nil? - next if database_index.statement == structure_sql_index.statement - - build_inconsistency(self.class, structure_sql_index, database_index) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb b/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb deleted file mode 100644 index 9fbddbd3fcd..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_tables.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionTables < BaseValidator - ERROR_MESSAGE = "The table %s has a different column statement between structure.sql and database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - table_name = structure_sql_table.name - database_table = database.fetch_table_by_name(table_name) - - next unless database_table - - db_diffs, structure_diffs = column_diffs(database_table, structure_sql_table.columns) - - if db_diffs.any? - build_inconsistency(self.class, - SchemaObjects::Table.new(table_name, db_diffs), - SchemaObjects::Table.new(table_name, structure_diffs)) - end - end - end - - private - - def column_diffs(db_table, columns) - db_diffs = [] - structure_diffs = [] - - columns.each do |column| - db_column = db_table.fetch_column_by_name(column.name) - - next unless db_column - - next if db_column.statement == column.statement - - db_diffs << db_column - structure_diffs << column - end - - [db_diffs, structure_diffs] - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb b/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb deleted file mode 100644 index 79ffe9a6a98..00000000000 --- a/lib/gitlab/database/schema_validation/validators/different_definition_triggers.rb +++ /dev/null @@ -1,24 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class DifferentDefinitionTriggers < BaseValidator - ERROR_MESSAGE = "The %s trigger has a different statement between structure.sql and database" - - def execute - structure_sql.triggers.filter_map do |structure_sql_trigger| - database_trigger = database.fetch_trigger_by_name(structure_sql_trigger.name) - - next if database_trigger.nil? - next if database_trigger.statement == structure_sql_trigger.statement - - build_inconsistency(self.class, structure_sql_trigger, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb deleted file mode 100644 index 887e86c7bfd..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_foreign_keys.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraForeignKeys < BaseValidator - ERROR_MESSAGE = "The foreign key %s is present in the database, but not in the structure.sql file" - - def execute - database.foreign_keys.filter_map do |database_fk| - next if structure_sql.foreign_key_exists?(database_fk.name) - - build_inconsistency(self.class, nil, database_fk) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb b/lib/gitlab/database/schema_validation/validators/extra_indexes.rb deleted file mode 100644 index c8d3749894b..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_indexes.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraIndexes < BaseValidator - ERROR_MESSAGE = "The index %s is present in the database, but not in the structure.sql file" - - def execute - database.indexes.filter_map do |database_index| - next if structure_sql.index_exists?(database_index.name) - - build_inconsistency(self.class, nil, database_index) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb b/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb deleted file mode 100644 index 823b01cf808..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_table_columns.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTableColumns < BaseValidator - ERROR_MESSAGE = "The table %s has columns present in the database, but not in the structure.sql file" - - def execute - database.tables.filter_map do |database_table| - table_name = database_table.name - structure_sql_table = structure_sql.fetch_table_by_name(table_name) - - next unless structure_sql_table - - inconsistencies = database_table.columns.filter_map do |database_table_column| - next if structure_sql_table.column_exists?(database_table_column.name) - - database_table_column - end - - if inconsistencies.any? - build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_tables.rb b/lib/gitlab/database/schema_validation/validators/extra_tables.rb deleted file mode 100644 index 99e98eb8f67..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_tables.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTables < BaseValidator - ERROR_MESSAGE = "The table %s is present in the database, but not in the structure.sql file" - - def execute - database.tables.filter_map do |database_table| - next if structure_sql.table_exists?(database_table.name) - - build_inconsistency(self.class, nil, database_table) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb b/lib/gitlab/database/schema_validation/validators/extra_triggers.rb deleted file mode 100644 index 37dcbc53e2e..00000000000 --- a/lib/gitlab/database/schema_validation/validators/extra_triggers.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class ExtraTriggers < BaseValidator - ERROR_MESSAGE = "The trigger %s is present in the database, but not in the structure.sql file" - - def execute - database.triggers.filter_map do |database_trigger| - next if structure_sql.trigger_exists?(database_trigger.name) - - build_inconsistency(self.class, nil, database_trigger) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb b/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb deleted file mode 100644 index b20f8474426..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_foreign_keys.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingForeignKeys < BaseValidator - ERROR_MESSAGE = "The foreign key %s is missing from the database" - - def execute - structure_sql.foreign_keys.filter_map do |structure_sql_fk| - next if database.foreign_key_exists?(structure_sql_fk.name) - - build_inconsistency(self.class, structure_sql_fk, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb b/lib/gitlab/database/schema_validation/validators/missing_indexes.rb deleted file mode 100644 index 7f81aaccf0f..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_indexes.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingIndexes < BaseValidator - ERROR_MESSAGE = "The index %s is missing from the database" - - def execute - structure_sql.indexes.filter_map do |structure_sql_index| - next if database.index_exists?(structure_sql_index.name) - - build_inconsistency(self.class, structure_sql_index, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb b/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb deleted file mode 100644 index b49d53823ee..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_table_columns.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTableColumns < BaseValidator - ERROR_MESSAGE = "The table %s has columns missing from the database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - table_name = structure_sql_table.name - database_table = database.fetch_table_by_name(table_name) - - next unless database_table - - inconsistencies = structure_sql_table.columns.filter_map do |structure_table_column| - next if database_table.column_exists?(structure_table_column.name) - - structure_table_column - end - - if inconsistencies.any? - build_inconsistency(self.class, nil, SchemaObjects::Table.new(table_name, inconsistencies)) - end - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_tables.rb b/lib/gitlab/database/schema_validation/validators/missing_tables.rb deleted file mode 100644 index f1c9383487d..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_tables.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTables < BaseValidator - ERROR_MESSAGE = "The table %s is missing from the database" - - def execute - structure_sql.tables.filter_map do |structure_sql_table| - next if database.table_exists?(structure_sql_table.name) - - build_inconsistency(self.class, structure_sql_table, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb b/lib/gitlab/database/schema_validation/validators/missing_triggers.rb deleted file mode 100644 index 36236463bbf..00000000000 --- a/lib/gitlab/database/schema_validation/validators/missing_triggers.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Database - module SchemaValidation - module Validators - class MissingTriggers < BaseValidator - ERROR_MESSAGE = "The trigger %s is missing from the database" - - def execute - structure_sql.triggers.filter_map do |structure_sql_trigger| - next if database.trigger_exists?(structure_sql_trigger.name) - - build_inconsistency(self.class, structure_sql_trigger, nil) - end - end - end - end - end - end -end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index 02e0da022f9..aa880b709fe 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -13,7 +13,7 @@ module Gitlab end def unlock_writes - Gitlab::Database::EachDatabase.each_database_connection do |connection, database_name| + Gitlab::Database::EachDatabase.each_connection do |connection, database_name| tables_to_lock(connection) do |table_name, schema_name| # TODO: https://gitlab.com/gitlab-org/gitlab/-/issues/366834 next if schema_name.in? GITLAB_SCHEMAS_TO_IGNORE @@ -28,7 +28,7 @@ module Gitlab # It locks the tables on the database where they don't belong. Also it unlocks the tables # on the database where they belong def lock_writes - Gitlab::Database::EachDatabase.each_database_connection(include_shared: false) do |connection, database_name| + Gitlab::Database::EachDatabase.each_connection(include_shared: false) do |connection, database_name| schemas_for_connection = Gitlab::Database.gitlab_schemas_for_connection(connection) tables_to_lock(connection) do |table_name, schema_name| diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 18ff7c28e17..31b214a4af9 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -42,8 +42,10 @@ module Gitlab with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do if Gitlab::Redis::ClusterUtil.cluster?(redis) - Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| - keys.each { |key| pipeline.get(key) } + redis.with_readonly_pipeline do + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + keys.each { |key| pipeline.get(key) } + end end else redis.mget(keys) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index 215ba77db13..5d0e6ea61e1 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -32,6 +32,9 @@ module Gitlab def execute raise ProjectNotFound if project.nil? + # Verification emails should never create issues + return if handled_custom_email_address_verification? + create_issue_or_note if from_address @@ -70,6 +73,27 @@ module Gitlab attr_reader :project_id, :project_path, :service_desk_key + def contains_custom_email_address_verification_subaddress? + return false unless Feature.enabled?(:service_desk_custom_email, project) + + # Verification email only has one recipient + mail.to.first.include?(ServiceDeskSetting::CUSTOM_EMAIL_VERIFICATION_SUBADDRESS) + end + + def handled_custom_email_address_verification? + return false unless contains_custom_email_address_verification_subaddress? + + ::ServiceDesk::CustomEmailVerifications::UpdateService.new( + project: project, + current_user: nil, + params: { + mail: mail + } + ).execute + + true + end + def project_from_key return unless match = service_desk_key.match(PROJECT_KEY_PATTERN) diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index 51d250ea98c..ee11105537b 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -8,7 +8,7 @@ module Gitlab class Receiver include Gitlab::Utils::StrongMemoize - RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + RECEIVED_HEADER_REGEX = /for\s+\<([^<]+)\>/.freeze # Errors that are purely from users and not anything we can control USER_ERRORS = [ diff --git a/lib/gitlab/error_tracking/error_repository.rb b/lib/gitlab/error_tracking/error_repository.rb index 3871305c9c5..79557838abf 100644 --- a/lib/gitlab/error_tracking/error_repository.rb +++ b/lib/gitlab/error_tracking/error_repository.rb @@ -15,12 +15,7 @@ module Gitlab # # @return [self] def self.build(project) - strategy = - if Feature.enabled?(:gitlab_error_tracking, project) - OpenApiStrategy.new(project) - else - ActiveRecordStrategy.new(project) - end + strategy = OpenApiStrategy.new(project) new(strategy) end diff --git a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb b/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb deleted file mode 100644 index 01e7fbda384..00000000000 --- a/lib/gitlab/error_tracking/error_repository/active_record_strategy.rb +++ /dev/null @@ -1,117 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ErrorTracking - class ErrorRepository - class ActiveRecordStrategy - def initialize(project) - @project = project - end - - def report_error( - name:, description:, actor:, platform:, - environment:, level:, occurred_at:, payload: - ) - error = project_errors.report_error( - name: name, # Example: ActionView::MissingTemplate - description: description, # Example: Missing template posts/show in... - actor: actor, # Example: PostsController#show - platform: platform, # Example: ruby - timestamp: occurred_at - ) - - # The payload field contains all the data on error including stacktrace in jsonb. - # Together with occurred_at these are 2 main attributes that we need to save here. - error.events.create!( - environment: environment, - description: description, - level: level, - occurred_at: occurred_at, - payload: payload - ) - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def find_error(id) - project_error(id).to_sentry_detailed_error - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def list_errors(filters:, query:, sort:, limit:, cursor:) - errors = project_errors - errors = filter_by_status(errors, filters[:status]) - errors = sort(errors, sort) - errors = errors.keyset_paginate(cursor: cursor, per_page: limit) - # query is not supported - - pagination = ErrorRepository::Pagination.new(errors.cursor_for_next_page, errors.cursor_for_previous_page) - - [errors.map(&:to_sentry_error), pagination] - end - - def last_event_for(id) - project_error(id).last_event&.to_sentry_error_event - rescue ActiveRecord::ActiveRecordError => e - handle_exceptions(e) - end - - def update_error(id, **attributes) - project_error(id).update(attributes) - end - - def dsn_url(public_key) - gitlab = Settings.gitlab - - custom_port = Settings.gitlab_on_standard_port? ? nil : ":#{gitlab.port}" - - base_url = [ - gitlab.protocol, - "://", - public_key, - '@', - gitlab.host, - custom_port, - gitlab.relative_url_root - ].join('') - - "#{base_url}/api/v4/error_tracking/collector/#{project.id}" - end - - private - - attr_reader :project - - def project_errors - ::ErrorTracking::Error.where(project: project) # rubocop:disable CodeReuse/ActiveRecord - end - - def project_error(id) - project_errors.find(id) - end - - def filter_by_status(errors, status) - return errors unless ::ErrorTracking::Error.statuses.key?(status) - - errors.for_status(status) - end - - def sort(errors, sort) - return errors.order_id_desc unless sort - - errors.sort_by_attribute(sort) - end - - def handle_exceptions(exception) - case exception - when ActiveRecord::RecordInvalid - raise RecordInvalidError, exception.message - else - raise DatabaseError, exception.message - end - end - end - end - end -end diff --git a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb index ab0df39e512..c141398bee0 100644 --- a/lib/gitlab/error_tracking/processor/grpc_error_processor.rb +++ b/lib/gitlab/error_tracking/processor/grpc_error_processor.rb @@ -6,7 +6,8 @@ module Gitlab module GrpcErrorProcessor extend Gitlab::ErrorTracking::Processor::Concerns::ProcessesExceptions - DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:(.*)') + # Braces added by gRPC Ruby code: https://github.com/grpc/grpc/blob/0e38b075ffff72ab2ad5326e3f60ba6dcc234f46/src/ruby/lib/grpc/errors.rb#L46 + DEBUG_ERROR_STRING_REGEX = RE2('(.*) debug_error_string:\{(.*)\}') class << self def call(event) diff --git a/lib/gitlab/exception_log_formatter.rb b/lib/gitlab/exception_log_formatter.rb index 52ad67d6f8b..a32f837eee9 100644 --- a/lib/gitlab/exception_log_formatter.rb +++ b/lib/gitlab/exception_log_formatter.rb @@ -21,6 +21,10 @@ module Gitlab payload['exception.cause_class'] = exception.cause.class.name end + if gitaly_metadata = find_gitaly_metadata(exception) + payload['exception.gitaly'] = gitaly_metadata.to_s + end + if sql = find_sql(exception) payload['exception.sql'] = sql end @@ -35,6 +39,16 @@ module Gitlab end end + def find_gitaly_metadata(exception) + if exception.is_a?(::Gitlab::Git::BaseError) + exception.metadata + elsif exception.is_a?(::GRPC::BadStatus) + exception.metadata[::Gitlab::Git::BaseError::METADATA_KEY] + elsif exception.cause.present? + find_gitaly_metadata(exception.cause) + end + end + private def normalize_query(sql) diff --git a/lib/gitlab/git/base_error.rb b/lib/gitlab/git/base_error.rb index 0b0fdef54cc..330e947844c 100644 --- a/lib/gitlab/git/base_error.rb +++ b/lib/gitlab/git/base_error.rb @@ -4,6 +4,7 @@ require 'grpc' module Gitlab module Git class BaseError < StandardError + METADATA_KEY = :gitaly_error_metadata DEBUG_ERROR_STRING_REGEX = /(.*?) debug_error_string:.*$/m.freeze GRPC_CODES = { '0' => 'ok', @@ -25,12 +26,15 @@ module Gitlab '16' => 'unauthenticated' }.freeze - attr_reader :status, :code, :service + attr_reader :status, :code, :service, :metadata def initialize(msg = nil) super && return if msg.nil? - set_grpc_error_code(msg) if msg.is_a?(::GRPC::BadStatus) + if msg.is_a?(::GRPC::BadStatus) + set_grpc_error_code(msg) + set_grpc_error_metadata(msg) + end super(build_raw_message(msg)) end @@ -46,6 +50,10 @@ module Gitlab @code = GRPC_CODES[@status.to_s] @service = 'git' end + + def set_grpc_error_metadata(grpc_error) + @metadata = grpc_error.metadata.fetch(METADATA_KEY, {}).clone + end end end end diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 11eb0a584ab..c0601c7795c 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -320,7 +320,7 @@ module Gitlab end def first_ref_by_oid(repo) - ref = repo.refs_by_oid(oid: id, limit: 1)&.first + ref = repo.refs_by_oid(oid: id, limit: 1).first return unless ref diff --git a/lib/gitlab/git/finders/refs_finder.rb b/lib/gitlab/git/finders/refs_finder.rb new file mode 100644 index 00000000000..a0117bc0fa9 --- /dev/null +++ b/lib/gitlab/git/finders/refs_finder.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Gitlab + module Git + module Finders + class RefsFinder + attr_reader :repository, :search, :ref_type + + UnknownRefTypeError = Class.new(StandardError) + + def initialize(repository, search:, ref_type:) + @repository = repository + @search = search + @ref_type = ref_type + end + + def execute + pattern = [prefix, search, "*"].compact.join + + repository.list_refs( + [pattern] + ) + end + + private + + def prefix + case ref_type + when :branches + Gitlab::Git::BRANCH_REF_PREFIX + when :tags + Gitlab::Git::TAG_REF_PREFIX + else + raise UnknownRefTypeError, "ref_type must be one of [:branches, :tags]" + end + end + end + end + end +end diff --git a/lib/gitlab/git/hook_env.rb b/lib/gitlab/git/hook_env.rb index f93ab19fc65..2524d4c4cfb 100644 --- a/lib/gitlab/git/hook_env.rb +++ b/lib/gitlab/git/hook_env.rb @@ -14,7 +14,7 @@ module Gitlab # # This class is thread-safe via RequestStore. class HookEnv - WHITELISTED_VARIABLES = %w[ + ALLOWLISTED_VARIABLES = %w[ GIT_OBJECT_DIRECTORY_RELATIVE GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE ].freeze @@ -25,7 +25,7 @@ module Gitlab raise "missing gl_repository" if gl_repository.blank? Gitlab::SafeRequestStore[:gitlab_git_env] ||= {} - Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = whitelist_git_env(env) + Gitlab::SafeRequestStore[:gitlab_git_env][gl_repository] = allowlist_git_env(env) end def self.all(gl_repository) @@ -46,8 +46,8 @@ module Gitlab env end - def self.whitelist_git_env(env) - env.select { |key, _| WHITELISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access + def self.allowlist_git_env(env) + env.select { |key, _| ALLOWLISTED_VARIABLES.include?(key.to_s) }.with_indifferent_access end end end diff --git a/lib/gitlab/git/keep_around.rb b/lib/gitlab/git/keep_around.rb index 38f0e47c4c7..5835a7001af 100644 --- a/lib/gitlab/git/keep_around.rb +++ b/lib/gitlab/git/keep_around.rb @@ -19,6 +19,8 @@ module Gitlab end def execute(shas) + return if disabled? + shas.uniq.each do |sha| next unless sha.present? && commit_by(oid: sha) @@ -32,6 +34,8 @@ module Gitlab end def kept_around?(sha) + return true if disabled? + ref_exists?(keep_around_ref_name(sha)) end @@ -40,6 +44,11 @@ module Gitlab private + def disabled? + Feature.enabled?(:disable_keep_around_refs, @repository, type: :ops) || + (@repository.project && Feature.enabled?(:disable_keep_around_refs, @repository.project, type: :ops)) + end + def keep_around_ref_name(sha) "refs/#{::Repository::REF_KEEP_AROUND}/#{sha}" end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index ed45d3eb030..71be986882c 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -337,9 +337,15 @@ module Gitlab # Return repo size in megabytes def size - size = gitaly_repository_client.repository_size + if Feature.enabled?(:use_repository_info_for_repository_size) + bytes = gitaly_repository_client.repository_info.size - (size.to_f / 1024).round(2) + (bytes.to_f / 1024 / 1024).round(2) + else + kilobytes = gitaly_repository_client.repository_size + + (kilobytes.to_f / 1024).round(2) + end end # Return git object directory size in bytes @@ -401,11 +407,12 @@ module Gitlab newrevs = newrevs.uniq.sort - @new_blobs ||= Hash.new do |h, revs| - h[revs] = blobs(['--not', '--all', '--not'] + newrevs, with_paths: true, dynamic_timeout: dynamic_timeout) - end - - @new_blobs[newrevs] + @new_blobs ||= {} + @new_blobs[newrevs] ||= blobs( + ['--not', '--all', '--not'] + newrevs, + with_paths: true, + dynamic_timeout: dynamic_timeout + ).to_a end # List blobs reachable via a set of revisions. Supports the @@ -554,10 +561,10 @@ module Gitlab # Limit of 0 means there is no limit. def refs_by_oid(oid:, limit: 0, ref_patterns: nil) wrapped_gitaly_errors do - gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) + gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) || [] end rescue CommandError, TypeError, NoRepository - nil + [] end # Returns url for submodule diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index df3d8165ef2..140dc791135 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -26,6 +26,11 @@ module Gitlab repository.gitaly_commit_client.tree_entries( repository, sha, path, recursive, skip_flat_paths, pagination_params) end + + # Incorrect revision or path could lead to index error. + # We silently handle such errors by returning an empty set of entries and cursor. + rescue Gitlab::Git::Index::IndexError + [[], nil] end private diff --git a/lib/gitlab/gitaly_client/call.rb b/lib/gitlab/gitaly_client/call.rb index 3fe3702cfe1..37d3921d6d5 100644 --- a/lib/gitlab/gitaly_client/call.rb +++ b/lib/gitlab/gitaly_client/call.rb @@ -32,6 +32,8 @@ module Gitlab end rescue StandardError => err store_timings + set_gitaly_error_metadata(err) if err.is_a?(::GRPC::BadStatus) + raise err end @@ -44,6 +46,9 @@ module Gitlab yielder.yield(value) end + rescue ::GRPC::BadStatus => err + set_gitaly_error_metadata(err) + raise err ensure store_timings end @@ -73,6 +78,15 @@ module Gitlab backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller) ) end + + def set_gitaly_error_metadata(err) + err.metadata[::Gitlab::Git::BaseError::METADATA_KEY] = { + storage: @storage, + address: ::Gitlab::GitalyClient.address(@storage), + service: @service, + rpc: @rpc + } + end end end end diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index aa25fd3589a..c10f780665c 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -531,14 +531,24 @@ module Gitlab request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) - signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } + signatures = Hash.new do |h, k| + h[k] = { + signature: +''.b, + signed_text: +''.b, + signer: :SIGNER_UNSPECIFIED + } + end + current_commit_id = nil response.each do |message| current_commit_id = message.commit_id if message.commit_id.present? - signatures[current_commit_id].first << message.signature - signatures[current_commit_id].last << message.signed_text + signatures[current_commit_id][:signature] << message.signature + signatures[current_commit_id][:signed_text] << message.signed_text + + # The actual value is send once. All the other chunks send SIGNER_UNSPECIFIED + signatures[current_commit_id][:signer] = message.signer unless message.signer == :SIGNER_UNSPECIFIED end signatures @@ -585,9 +595,7 @@ module Gitlab end def call_commit_diff(request_params, options = {}) - request_params[:ignore_whitespace_change] = options.fetch(:ignore_whitespace_change, false) - - if Feature.enabled?(:add_ignore_all_white_spaces) && (request_params[:ignore_whitespace_change]) + if options.fetch(:ignore_whitespace_change, false) request_params[:whitespace_changes] = WHITESPACE_CHANGES['ignore_all_spaces'] end @@ -641,10 +649,6 @@ module Gitlab def find_changed_paths_request(commits, merge_commit_diff_mode) diff_mode = MERGE_COMMIT_DIFF_MODES[merge_commit_diff_mode] if Feature.enabled?(:merge_commit_diff_modes) - if Feature.disabled?(:find_changed_paths_new_format) - return Gitaly::FindChangedPathsRequest.new(repository: @gitaly_repo, commits: commits, merge_commit_diff_mode: diff_mode) - end - commit_requests = commits.map do |commit| Gitaly::FindChangedPathsRequest::Request.new( commit_request: Gitaly::FindChangedPathsRequest::Request::CommitRequest.new(commit_revision: commit) diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index bd6cc9105d9..67e135bb530 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -135,7 +135,7 @@ module Gitlab end end - def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:, allow_conflicts: false) + def user_merge_to_ref(user, source_sha:, branch:, target_ref:, message:, first_parent_ref:) request = Gitaly::UserMergeToRefRequest.new( repository: @gitaly_repo, source_sha: source_sha, @@ -144,7 +144,6 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, message: encode_binary(message), first_parent_ref: encode_binary(first_parent_ref), - allow_conflicts: allow_conflicts, timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 93d58710b0c..b5b7d94b4d0 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -52,6 +52,12 @@ module Gitlab response.size end + def repository_info + request = Gitaly::RepositoryInfoRequest.new(repository: @gitaly_repo) + + gitaly_client_call(@storage, :repository_service, :repository_info, request, timeout: GitalyClient.long_timeout) + end + def get_object_directory_size request = Gitaly::GetObjectDirectorySizeRequest.new(repository: @gitaly_repo) response = gitaly_client_call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) diff --git a/lib/gitlab/github_import.rb b/lib/gitlab/github_import.rb index 9556a9e98ba..24e77363e1b 100644 --- a/lib/gitlab/github_import.rb +++ b/lib/gitlab/github_import.rb @@ -8,12 +8,18 @@ module Gitlab def self.new_client_for(project, token: nil, host: nil, parallel: true) token_to_use = token || project.import_data&.credentials&.fetch(:user) - Client.new( - token_to_use, + token_pool = project.import_data&.credentials&.dig(:additional_access_tokens) + options = { host: host.presence || self.formatted_import_url(project), per_page: self.per_page(project), parallel: parallel - ) + } + + if token_pool + ClientPool.new(token_pool: token_pool, **options) + else + Client.new(token_to_use, **options) + end end # Returns the ID of the ghost user. diff --git a/lib/gitlab/github_import/client_pool.rb b/lib/gitlab/github_import/client_pool.rb new file mode 100644 index 00000000000..e8414942d1b --- /dev/null +++ b/lib/gitlab/github_import/client_pool.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + class ClientPool + delegate_missing_to :best_client + + def initialize(token_pool:, per_page:, parallel:, host: nil) + @token_pool = token_pool + @host = host + @per_page = per_page + @parallel = parallel + end + + # Returns the client with the most remaining requests, or the client with + # the closest rate limit reset time, if all clients are rate limited. + def best_client + clients_with_requests_remaining = clients.select(&:requests_remaining?) + + return clients_with_requests_remaining.max_by(&:remaining_requests) if clients_with_requests_remaining.any? + + clients.min_by(&:rate_limit_resets_in) + end + + private + + def clients + @clients ||= @token_pool.map do |token| + Client.new( + token, + host: @host, + per_page: @per_page, + parallel: @parallel + ) + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/issue_importer.rb b/lib/gitlab/github_import/importer/issue_importer.rb index b477468d327..a537841ecf3 100644 --- a/lib/gitlab/github_import/importer/issue_importer.rb +++ b/lib/gitlab/github_import/importer/issue_importer.rb @@ -31,7 +31,7 @@ module Gitlab if (issue_id = create_issue) create_assignees(issue_id) issuable_finder.cache_database_id(issue_id) - update_search_data(issue_id) if Feature.enabled?(:issues_full_text_search) + update_search_data(issue_id) end end end diff --git a/lib/gitlab/github_import/settings.rb b/lib/gitlab/github_import/settings.rb index 0b883de8ed0..73a5f49a9e3 100644 --- a/lib/gitlab/github_import/settings.rb +++ b/lib/gitlab/github_import/settings.rb @@ -56,8 +56,16 @@ module Gitlab def write(user_settings) user_settings = user_settings.to_h.with_indifferent_access - optional_stages = fetch_stages_from_params(user_settings) - import_data = project.create_or_update_import_data(data: { optional_stages: optional_stages }) + optional_stages = fetch_stages_from_params(user_settings[:optional_stages]) + credentials = project.import_data&.credentials&.merge( + additional_access_tokens: user_settings[:additional_access_tokens] + ) + + import_data = project.create_or_update_import_data( + data: { optional_stages: optional_stages }, + credentials: credentials + ) + import_data.save! end @@ -74,6 +82,8 @@ module Gitlab attr_reader :project def fetch_stages_from_params(user_settings) + user_settings = user_settings.to_h.with_indifferent_access + OPTIONAL_STAGES.keys.to_h do |stage_name| enabled = Gitlab::Utils.to_boolean(user_settings[stage_name], default: false) [stage_name, enabled] diff --git a/lib/gitlab/github_import/user_finder.rb b/lib/gitlab/github_import/user_finder.rb index dd71edbd205..57365ebe206 100644 --- a/lib/gitlab/github_import/user_finder.rb +++ b/lib/gitlab/github_import/user_finder.rb @@ -28,9 +28,6 @@ module Gitlab EMAIL_FOR_USERNAME_CACHE_KEY = 'github-import/user-finder/email-for-username/%s' - # The base cache key to use for caching inexistence of GitHub usernames. - INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY = 'github-import/user-finder/inexistence-of-username/%s' - # project - An instance of `Project` # client - An instance of `Gitlab::GithubImport::Client` def initialize(project, client) @@ -112,18 +109,24 @@ module Gitlab id_for_github_id(id) || id_for_github_email(email) end + # Find the public email of a given username in GitHub. The public email is cached to avoid multiple calls to + # GitHub. In case the username does not exist or the public email is nil, a blank value is cached to also prevent + # multiple calls to GitHub. + # + # @return [String] If public email is found + # @return [Nil] If public email or username does not exist def email_for_github_username(username) cache_key = EMAIL_FOR_USERNAME_CACHE_KEY % username email = Gitlab::Cache::Import::Caching.read(cache_key) - if email.blank? && !github_username_inexists?(username) + if email.nil? user = client.user(username) - email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email], timeout: timeout(user[:email])) if user + email = Gitlab::Cache::Import::Caching.write(cache_key, user[:email].to_s, timeout: timeout(user[:email])) end - email + email.presence rescue ::Octokit::NotFound - cache_github_username_inexistence(username) + Gitlab::Cache::Import::Caching.write(cache_key, '') nil end @@ -196,18 +199,6 @@ module Gitlab Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT end end - - def github_username_inexists?(username) - cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username - - Gitlab::Cache::Import::Caching.read(cache_key) == 'true' - end - - def cache_github_username_inexistence(username) - cache_key = INEXISTENCE_OF_GITHUB_USERNAME_CACHE_KEY % username - - Gitlab::Cache::Import::Caching.write(cache_key, true) - end end end end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index 9eeea7336b5..ff171c24549 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -75,6 +75,7 @@ module Gitlab # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:gitlab_duo, current_user) + push_frontend_feature_flag(:custom_emoji) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/gpg/commit.rb b/lib/gitlab/gpg/commit.rb index a03aeb9c293..1fc95181767 100644 --- a/lib/gitlab/gpg/commit.rb +++ b/lib/gitlab/gpg/commit.rb @@ -87,6 +87,7 @@ module Gitlab end def verification_status(gpg_key) + return :verified_system if verified_by_gitlab? return :multiple_signatures if multiple_signatures? return :unknown_key unless gpg_key return :unverified_key unless gpg_key.verified? @@ -101,6 +102,15 @@ module Gitlab end end + # If a commit is signed by Gitaly, the Gitaly returns `SIGNER_SYSTEM` as a signer + # In order to calculate it, the signature is Verified using the Gitaly's public key: + # https://gitlab.com/gitlab-org/gitaly/-/blob/v16.2.0-rc2/internal/gitaly/service/commit/commit_signatures.go#L63 + # + # It is safe to skip verification step if the commit has been signed by Gitaly + def verified_by_gitlab? + signer == :SIGNER_SYSTEM + end + def user_infos(gpg_key) gpg_key&.verified_user_infos&.first || gpg_key&.user_infos&.first || {} end diff --git a/lib/gitlab/grape_logging/loggers/response_logger.rb b/lib/gitlab/grape_logging/loggers/response_logger.rb index 767c282d62e..b87566a62b0 100644 --- a/lib/gitlab/grape_logging/loggers/response_logger.rb +++ b/lib/gitlab/grape_logging/loggers/response_logger.rb @@ -5,8 +5,6 @@ module Gitlab module Loggers class ResponseLogger < ::GrapeLogging::Loggers::Base def parameters(_, response) - return {} unless Feature.enabled?(:log_response_length) - response_bytes = 0 case response diff --git a/lib/gitlab/graphql/generic_tracing.rb b/lib/gitlab/graphql/generic_tracing.rb deleted file mode 100644 index dc3f6574631..00000000000 --- a/lib/gitlab/graphql/generic_tracing.rb +++ /dev/null @@ -1,71 +0,0 @@ -# frozen_string_literal: true - -# This class is used as a hook to observe graphql runtime events. From this -# hook both gitlab metrics and opentracking measurements are generated - -module Gitlab - module Graphql - class GenericTracing < GraphQL::Tracing::PlatformTracing - self.platform_keys = { - 'lex' => 'graphql.lex', - 'parse' => 'graphql.parse', - 'validate' => 'graphql.validate', - 'analyze_query' => 'graphql.analyze', - 'analyze_multiplex' => 'graphql.analyze', - 'execute_multiplex' => 'graphql.execute', - 'execute_query' => 'graphql.execute', - 'execute_query_lazy' => 'graphql.execute', - 'execute_field' => 'graphql.execute', - 'execute_field_lazy' => 'graphql.execute' - } - - def platform_field_key(type, field) - "#{type.name}.#{field.name}" - end - - def platform_authorized_key(type) - "#{type.graphql_name}.authorized" - end - - def platform_resolve_type_key(type) - "#{type.graphql_name}.resolve_type" - end - - def platform_trace(platform_key, key, data, &block) - tags = { platform_key: platform_key, key: key } - start = Gitlab::Metrics::System.monotonic_time - - with_labkit_tracing(tags, &block) - ensure - duration = Gitlab::Metrics::System.monotonic_time - start - - graphql_duration_seconds.observe(tags, duration) unless deactivated? - end - - private - - def deactivated? - Feature.enabled?(:graphql_generic_tracing_metrics_deactivate) - end - - def with_labkit_tracing(tags, &block) - return yield unless Labkit::Tracing.enabled? - - name = "#{tags[:platform_key]}.#{tags[:key]}" - span_tags = { - 'component' => 'web', - 'span.kind' => 'server' - }.merge(tags.stringify_keys) - - Labkit::Tracing.with_tracing(operation_name: name, tags: span_tags, &block) - end - - def graphql_duration_seconds - @graphql_duration_seconds ||= Gitlab::Metrics.histogram( - :graphql_duration_seconds, - 'GraphQL execution time' - ) - end - end - end -end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index b112740c4ad..8ca88859b22 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,6 +13,7 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) + groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/hook_data/emoji_builder.rb b/lib/gitlab/hook_data/emoji_builder.rb new file mode 100644 index 00000000000..673eb516e43 --- /dev/null +++ b/lib/gitlab/hook_data/emoji_builder.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module HookData + class EmojiBuilder < BaseBuilder + SAFE_HOOK_ATTRIBUTES = %i[ + user_id + created_at + id + name + awardable_type + awardable_id + updated_at + ].freeze + + alias_method :award_emoji, :object + + def build + award_emoji + .attributes + .with_indifferent_access + .slice(*SAFE_HOOK_ATTRIBUTES) + end + end + end +end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index 180ccf21264..fabc02af70a 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 31, - 'de' => 97, + 'da_DK' => 30, + 'de' => 99, 'en' => 100, 'eo' => 0, - 'es' => 30, + 'es' => 29, 'fil_PH' => 0, - 'fr' => 98, + 'fr' => 99, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 98, - 'ko' => 18, + 'ja' => 99, + 'ko' => 17, 'nb_NO' => 22, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 56, - 'ro_RO' => 82, + 'pt_BR' => 57, + 'ro_RO' => 80, 'ru' => 23, 'si_LK' => 10, 'tr_TR' => 9, 'uk' => 53, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 100 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/import_export/group/import_export.yml b/lib/gitlab/import_export/group/import_export.yml index c2a1a1f8575..7a91cfb340a 100644 --- a/lib/gitlab/import_export/group/import_export.yml +++ b/lib/gitlab/import_export/group/import_export.yml @@ -7,12 +7,10 @@ tree: group: - :milestones - :badges - - labels: - - :priorities + - :labels - boards: - lists: - - label: - - :priorities + - :label - :board - members: - :user @@ -126,8 +124,7 @@ ee: - boards: - :board_assignee - :milestone - - labels: - - :priorities + - :labels - lists: - milestone: - events: diff --git a/lib/gitlab/import_export/group/relation_factory.rb b/lib/gitlab/import_export/group/relation_factory.rb index 1b8436c4ed9..664ef5358ef 100644 --- a/lib/gitlab/import_export/group/relation_factory.rb +++ b/lib/gitlab/import_export/group/relation_factory.rb @@ -6,7 +6,6 @@ module Gitlab class RelationFactory < Base::RelationFactory OVERRIDES = { labels: :group_labels, - priorities: :label_priorities, label: :group_label, parent: :epic, iterations_cadences: 'Iterations::Cadence' diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 410e918649b..5986c5de441 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -44,6 +44,7 @@ tree: - :zoom_meetings - :sentry_issue - :award_emoji + - :work_item_type - snippets: - :award_emoji - notes: @@ -771,6 +772,8 @@ included_attributes: - :source_commit - :close_after_error_tracking_resolve - :close_auto_resolve_prometheus_alert + work_item_type: + - :base_type # Do not include the following attributes for the models specified. excluded_attributes: @@ -1101,6 +1104,15 @@ excluded_attributes: - :roll_over - :description - :sequence + work_item_type: + - :id + - :cached_markdown_version + - :name + - :description + - :description_html + - :icon_name + - :namespace_id + - :updated_at methods: project: diff --git a/lib/gitlab/import_export/project/object_builder.rb b/lib/gitlab/import_export/project/object_builder.rb index ac28ae6bfe0..5534a0f2aa4 100644 --- a/lib/gitlab/import_export/project/object_builder.rb +++ b/lib/gitlab/import_export/project/object_builder.rb @@ -24,6 +24,7 @@ module Gitlab return if group_relation_without_group? return find_diff_commit_user if diff_commit_user? return find_diff_commit if diff_commit? + return find_work_item_type if work_item_type? super end @@ -142,6 +143,10 @@ module Gitlab klass == MergeRequestDiffCommit end + def work_item_type? + klass == ::WorkItems::Type + end + # If an existing group milestone used the IID # claim the IID back and set the group milestone to use one available # This is necessary to fix situations like the following: @@ -166,6 +171,18 @@ module Gitlab def group_level_object? epic? end + + def find_work_item_type + base_type = @attributes['base_type'] + + find_with_cache([::WorkItems::Type, base_type]) do + if ::WorkItems::Type.base_types.key?(base_type) + ::WorkItems::Type.default_by_type(base_type) + else + ::WorkItems::Type.default_issue_type + end + end + end end end end diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb index 8c673acdd1a..7af65235492 100644 --- a/lib/gitlab/import_export/project/relation_factory.rb +++ b/lib/gitlab/import_export/project/relation_factory.rb @@ -16,6 +16,8 @@ module Gitlab bridges: 'Ci::Bridge', runners: 'Ci::Runner', pipeline_metadata: 'Ci::PipelineMetadata', + external_pull_request: 'Ci::ExternalPullRequest', + external_pull_requests: 'Ci::ExternalPullRequest', hooks: 'ProjectHook', merge_access_levels: 'ProtectedBranch::MergeAccessLevel', push_access_levels: 'ProtectedBranch::PushAccessLevel', @@ -39,7 +41,8 @@ module Gitlab metrics_setting: 'ProjectMetricsSetting', commit_author: 'MergeRequest::DiffCommitUser', committer: 'MergeRequest::DiffCommitUser', - merge_request_diff_commits: 'MergeRequestDiffCommit' }.freeze + merge_request_diff_commits: 'MergeRequestDiffCommit', + work_item_type: 'WorkItems::Type' }.freeze BUILD_MODELS = %i[Ci::Build Ci::Bridge commit_status generic_commit_status].freeze @@ -61,11 +64,11 @@ module Gitlab epic ProjectCiCdSetting container_expiration_policy - external_pull_request - external_pull_requests + Ci::ExternalPullRequest DesignManagement::Design MergeRequest::DiffCommitUser MergeRequestDiffCommit + WorkItems::Type ].freeze def create @@ -90,7 +93,7 @@ module Gitlab when :notes, :Note then setup_note when :'Ci::Pipeline' then setup_pipeline when *BUILD_MODELS then setup_build - when :issues then setup_issue + when :issues then setup_work_item when :'Ci::PipelineSchedule' then setup_pipeline_schedule when :'ProtectedBranch::MergeAccessLevel' then setup_protected_branch_access_level when :'ProtectedBranch::PushAccessLevel' then setup_protected_branch_access_level @@ -166,8 +169,11 @@ module Gitlab end end - def setup_issue + def setup_work_item @relation_hash['relative_position'] = compute_relative_position + + issue_type = @relation_hash.delete('issue_type') + @relation_hash['work_item_type'] ||= ::WorkItems::Type.default_by_type(issue_type) if issue_type end def setup_release diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index cde83068de1..92bf2a826ff 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -2,21 +2,38 @@ module Gitlab module InternalEvents + UnknownEventError = Class.new(StandardError) + InvalidPropertyError = Class.new(StandardError) + InvalidMethodError = Class.new(StandardError) + class << self include Gitlab::Tracking::Helpers def track_event(event_name, **kwargs) - user_id = kwargs.delete(:user_id) - UsageDataCounters::HLLRedisCounter.track_event(event_name, values: user_id) + raise UnknownEventError, "Unknown event: #{event_name}" unless EventDefinitions.known_event?(event_name) + + unique_property = EventDefinitions.unique_property(event_name) + unique_method = :id + + unless kwargs.has_key?(unique_property) + raise InvalidPropertyError, "#{event_name} should be triggered with a named parameter '#{unique_property}'." + end + + unless kwargs[unique_property].respond_to?(unique_method) + raise InvalidMethodError, "'#{unique_property}' should have a '#{unique_method}' method." + end + + unique_value = kwargs[unique_property].public_send(unique_method) # rubocop:disable GitlabSecurity/PublicSend - project_id = kwargs.delete(:project_id) - namespace_id = kwargs.delete(:namespace_id) + UsageDataCounters::HLLRedisCounter.track_event(event_name, values: unique_value) - namespace = Namespace.find(namespace_id) if namespace_id + user = kwargs[:user] + project = kwargs[:project] + namespace = kwargs[:namespace] standard_context = Tracking::StandardContext.new( - project_id: project_id, - user_id: user_id, + project_id: project&.id, + user_id: user&.id, namespace_id: namespace&.id, plan_name: namespace&.actual_plan_name ).to_context @@ -27,6 +44,9 @@ module Gitlab ).to_context track_struct_event(event_name, contexts: [standard_context, service_ping_context]) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e, event_name: event_name, kwargs: kwargs) + nil end private diff --git a/lib/gitlab/internal_events/event_definitions.rb b/lib/gitlab/internal_events/event_definitions.rb new file mode 100644 index 00000000000..e1c9faa12de --- /dev/null +++ b/lib/gitlab/internal_events/event_definitions.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module InternalEvents + module EventDefinitions + InvalidMetricConfiguration = Class.new(StandardError) + + class << self + VALID_UNIQUE_VALUES = %w[user.id project.id namespace.id].freeze + + def clear_events + @events = nil + end + + def load_configurations + @events = load_metric_definitions + nil + end + + def unique_property(event_name) + unique_value = events[event_name]&.to_s + + raise(InvalidMetricConfiguration, "Unique property not defined for #{event_name}") unless unique_value + + unless VALID_UNIQUE_VALUES.include?(unique_value) + raise(InvalidMetricConfiguration, "Invalid unique value '#{unique_value}' for #{event_name}") + end + + unique_value.split('.').first.to_sym + end + + def known_event?(event_name) + events.key?(event_name) + end + + private + + def events + load_configurations if @events.nil? || Gitlab::Usage::MetricDefinition.metric_definitions_changed? + + @events + end + + def load_metric_definitions + all_events = {} + + Gitlab::Usage::MetricDefinition.all.each do |metric_definition| + next unless metric_definition.available? + + process_events(all_events, metric_definition.events) + rescue StandardError => e + Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) + end + + all_events + end + + def process_events(all_events, metric_events) + metric_events.each do |event_name, event_unique_attribute| + unless all_events[event_name] + all_events[event_name] = event_unique_attribute + next + end + + next if event_unique_attribute.nil? || event_unique_attribute == all_events[event_name] + + raise InvalidMetricConfiguration, + "The same event cannot have several unique properties defined. " \ + "Event: #{event_name}, unique values: #{event_unique_attribute}, #{all_events[event_name]}" + end + end + end + end + end +end diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index f1f6cc55a2b..12cc5f6e5dd 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -53,7 +53,7 @@ module Gitlab end def can_start_rebalance? - rebalance_in_progress? || too_many_rebalances_running? + rebalance_in_progress? || concurrent_rebalance_within_limit? end def cache_issue_ids(issue_ids) @@ -100,11 +100,11 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) - multi.expire(current_index_key, REDIS_EXPIRY_TIME) - multi.expire(current_project_key, REDIS_EXPIRY_TIME) - multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + redis.pipelined do |pipeline| + pipeline.expire(issue_ids_key, REDIS_EXPIRY_TIME) + pipeline.expire(current_index_key, REDIS_EXPIRY_TIME) + pipeline.expire(current_project_key, REDIS_EXPIRY_TIME) + pipeline.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) end end end @@ -113,16 +113,20 @@ module Gitlab def cleanup_cache value = "#{rebalanced_container_type}/#{rebalanced_container_id}" + # The clean up is done sequentially to be compatible with Redis Cluster + # Do not use a pipeline as it fans-out in a Redis-Cluster setting and forego ordering guarantees with_redis do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.del(issue_ids_key) - multi.del(current_index_key) - multi.del(current_project_key) - multi.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) - multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) - end - end + # srem followed by .del(issue_ids_key) to ensure that any subsequent redis errors would + # result in a no-op job retry since current_index_key still exists + redis.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) + redis.del(issue_ids_key) + + # delete current_index_key to ensure that subsequent redis errors would + # result in a fresh job retry + redis.del(current_index_key) + + # setting recently_finished_key last after job details is cleaned up + redis.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) end end @@ -159,7 +163,7 @@ module Gitlab attr_accessor :root_namespace, :projects, :rebalanced_container_type, :rebalanced_container_id - def too_many_rebalances_running? + def concurrent_rebalance_within_limit? concurrent_running_rebalances_count <= MAX_NUMBER_OF_CONCURRENT_REBALANCES end diff --git a/lib/gitlab/jwt_authenticatable.rb b/lib/gitlab/jwt_authenticatable.rb index 7c36bbf3426..d7a341b3ba2 100644 --- a/lib/gitlab/jwt_authenticatable.rb +++ b/lib/gitlab/jwt_authenticatable.rb @@ -13,8 +13,8 @@ module Gitlab module ClassMethods include Gitlab::Utils::StrongMemoize - def decode_jwt(encoded_message, jwt_secret = secret, issuer: nil, iat_after: nil) - options = { algorithm: 'HS256' } + def decode_jwt(encoded_message, jwt_secret = secret, algorithm: 'HS256', issuer: nil, iat_after: nil) + options = { algorithm: algorithm } options = options.merge(iss: issuer, verify_iss: true) if issuer.present? options = options.merge(verify_iat: true) if iat_after.present? diff --git a/lib/gitlab/kas/client.rb b/lib/gitlab/kas/client.rb index 43546d04087..fe244bd88a0 100644 --- a/lib/gitlab/kas/client.rb +++ b/lib/gitlab/kas/client.rb @@ -31,7 +31,7 @@ module Gitlab def list_agent_config_files(project:) request = Gitlab::Agent::ConfigurationProject::Rpc::ListAgentConfigFilesRequest.new( repository: repository(project), - gitaly_address: gitaly_address(project) + gitaly_info: gitaly_info(project) ) stub_for(:configuration_project) @@ -42,9 +42,11 @@ module Gitlab def send_git_push_event(project:) request = Gitlab::Agent::Notifications::Rpc::GitPushEventRequest.new( - project: Gitlab::Agent::Notifications::Rpc::Project.new( - id: project.id, - full_path: project.full_path + event: Gitlab::Agent::Event::GitPushEvent.new( + project: Gitlab::Agent::Event::Project.new( + id: project.id, + full_path: project.full_path + ) ) ) @@ -62,13 +64,15 @@ module Gitlab def repository(project) gitaly_repository = project.repository.gitaly_repository - Gitlab::Agent::Modserver::Repository.new(gitaly_repository.to_h) + Gitlab::Agent::Entity::GitalyRepository.new(gitaly_repository.to_h) end - def gitaly_address(project) + def gitaly_info(project) + gitaly_features = Feature::Gitaly.server_feature_flags connection_data = Gitlab::GitalyClient.connection_data(project.repository_storage) + .merge(features: gitaly_features) - Gitlab::Agent::Modserver::GitalyAddress.new(connection_data) + Gitlab::Agent::Entity::GitalyInfo.new(connection_data) end def kas_endpoint_url diff --git a/lib/gitlab/kas/user_access.rb b/lib/gitlab/kas/user_access.rb index 65ae399d826..587aa4803c6 100644 --- a/lib/gitlab/kas/user_access.rb +++ b/lib/gitlab/kas/user_access.rb @@ -9,11 +9,7 @@ module Gitlab class UserAccess class << self def enabled? - ::Gitlab::Kas.enabled? && ::Feature.enabled?(:kas_user_access) - end - - def enabled_for?(agent) - enabled? && ::Feature.enabled?(:kas_user_access_project, agent.project) + ::Gitlab::Kas.enabled? end def encrypt_public_session_id(data) diff --git a/lib/gitlab/lograge/custom_options.rb b/lib/gitlab/lograge/custom_options.rb index f8ec58cf217..9abad44b10e 100644 --- a/lib/gitlab/lograge/custom_options.rb +++ b/lib/gitlab/lograge/custom_options.rb @@ -36,10 +36,6 @@ module Gitlab payload[:feature_flag_states] = Feature.logged_states.map { |key, state| "#{key}:#{state ? 1 : 0}" } end - if Feature.disabled?(:log_response_length) - payload.delete(:response_bytes) - end - payload end end diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 3747431c6a7..81711be729e 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -4,6 +4,7 @@ module Gitlab module ManifestImport class Metadata EXPIRY_TIME = 1.week + KEY_PREFIX = 'manifest_import:metadata:user' attr_reader :user, :fallback @@ -14,11 +15,9 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.multi do |multi| - multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) - end + redis.multi do |multi| + multi.set(hashtag_key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(hashtag_key_for('group_id'), group_id, ex: EXPIRY_TIME) end end end @@ -37,13 +36,17 @@ module Gitlab private + def hashtag_key_for(field) + "#{KEY_PREFIX}:{#{user.id}}:#{field}" + end + def key_for(field) - "manifest_import:metadata:user:#{user.id}:#{field}" + "#{KEY_PREFIX}:#{user.id}:#{field}" end def redis_get(field) Gitlab::Redis::SharedState.with do |redis| - redis.get(key_for(field)) + redis.get(hashtag_key_for(field)) || redis.get(key_for(field)) end end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index f742cb82b8d..52260623c55 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -11,9 +11,11 @@ module Gitlab Gitlab::Redis::Cache.with do |r| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| - subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + r.with_readonly_pipeline do + Gitlab::Redis::CrossSlot::Pipeline.new(r).pipelined do |pipeline| + subjects.each do |subject| + results[subject.cache_key] = new(subject).read(pipeline) + end end end end diff --git a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb index 31d75225972..56a82d1df46 100644 --- a/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb +++ b/lib/gitlab/metrics/dashboard/stages/cluster_endpoint_inserter.rb @@ -7,59 +7,14 @@ module Gitlab class ClusterEndpointInserter < BaseStage def transform! verify_params - - for_metrics do |metric| - metric[:prometheus_endpoint_path] = endpoint_for_metric(metric) - end end private - def admin_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_admin_cluster_path( - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - - def endpoint_for_metric(metric) - case params[:cluster_type] - when :admin - admin_url(metric) - when :group - error!(_('Group is required when cluster_type is :group')) unless params[:group] - group_url(metric) - when :project - error!(_('Project is required when cluster_type is :project')) unless project - project_url(metric) - else - error!(_('Unrecognized cluster type')) - end - end - def error!(message) raise Errors::DashboardProcessingError, message end - def group_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_group_cluster_path( - params[:group], - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - - def project_url(metric) - Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path( - project, - params[:cluster], - proxy_path: query_type(metric), - query: query_for_metric(metric) - ) - end - def query_type(metric) metric[:query] ? :query : :query_range end diff --git a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb index 622b6adec7e..03370ae7370 100644 --- a/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb +++ b/lib/gitlab/metrics/dashboard/stages/grafana_formatter.rb @@ -53,8 +53,7 @@ module Gitlab { id: "#{metric[:legendFormat]}_#{idx}", query_range: format_query(metric), - label: replace_variables(metric[:legendFormat]), - prometheus_endpoint_path: prometheus_endpoint_for_metric(metric) + label: replace_variables(metric[:legendFormat]) }.compact end @@ -89,17 +88,6 @@ module Gitlab end end - # Endpoint which will return prometheus metric data - # for the metric - def prometheus_endpoint_for_metric(metric) - Gitlab::Routing.url_helpers.project_grafana_api_path( - project, - datasource_id: datasource[:id], - proxy_path: PROXY_PATH, - query: format_query(metric) - ) - end - # Reformats query for compatibility with prometheus api. def format_query(metric) expression = remove_new_lines(metric[:expr]) diff --git a/lib/gitlab/metrics/dashboard/url.rb b/lib/gitlab/metrics/dashboard/url.rb index bdd28744137..e7b901861ef 100644 --- a/lib/gitlab/metrics/dashboard/url.rb +++ b/lib/gitlab/metrics/dashboard/url.rb @@ -12,26 +12,6 @@ module Gitlab ANCHOR_PATTERN = '(?<anchor>\#[a-z0-9_-]+)?' DASH_PATTERN = '(?:/-)' - # Matches urls for a metrics dashboard. - # This regex needs to match the old metrics URL, the new metrics URL, - # and the dashboard URL (inline_metrics_redactor_filter.rb - # uses this regex to match against the dashboard URL.) - # - # EX - Old URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics - # OR - # New URL: https://<host>/<namespace>/<project>/-/metrics?environment=<env_id> - # OR - # dashboard URL: https://<host>/<namespace>/<project>/environments/<env_id>/metrics_dashboard - def metrics_regex - strong_memoize(:metrics_regex) do - regex_for_project_metrics( - %r{ - ( #{environment_metrics_regex} ) | ( #{non_environment_metrics_regex} ) - }x - ) - end - end - # Matches dashboard urls for a Grafana embed. # # EX - https://<host>/<namespace>/<project>/grafana/metrics_dashboard @@ -99,11 +79,6 @@ module Gitlab .symbolize_keys end - # Builds a metrics dashboard url based on the passed in arguments - def build_dashboard_url(...) - Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(...) - end - private def environment_metrics_regex diff --git a/lib/gitlab/metrics/sidekiq_slis.rb b/lib/gitlab/metrics/sidekiq_slis.rb index f28cf4ac967..748666f2200 100644 --- a/lib/gitlab/metrics/sidekiq_slis.rb +++ b/lib/gitlab/metrics/sidekiq_slis.rb @@ -8,16 +8,26 @@ module Gitlab "low" => 300, "throttled" => 300 }.freeze + QUEUEING_URGENCY_DURATIONS = { + "high" => 10, + "low" => 60, + "throttled" => Float::INFINITY # no queueing target duration for throttled urgency + }.freeze # workers without urgency attribute have "low" urgency by default in # WorkerAttributes.get_urgency, just mirroring it here DEFAULT_EXECUTION_URGENCY_DURATION = EXECUTION_URGENCY_DURATIONS["low"] + DEFAULT_QUEUEING_URGENCY_DURATION = QUEUEING_URGENCY_DURATIONS["low"] class << self - def initialize_slis!(possible_labels) + def initialize_execution_slis!(possible_labels) Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_execution, possible_labels) Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:sidekiq_execution, possible_labels) end + def initialize_queueing_slis!(possible_labels) + Gitlab::Metrics::Sli::Apdex.initialize_sli(:sidekiq_queueing, possible_labels) + end + def record_execution_apdex(labels, job_completion_duration) urgency_requirement = execution_duration_for_urgency(labels[:urgency]) Gitlab::Metrics::Sli::Apdex[:sidekiq_execution].increment( @@ -30,9 +40,21 @@ module Gitlab Gitlab::Metrics::Sli::ErrorRate[:sidekiq_execution].increment(labels: labels, error: error) end + def record_queueing_apdex(labels, queue_duration) + urgency_requirement = queueing_duration_for_urgency(labels[:urgency]) + Gitlab::Metrics::Sli::Apdex[:sidekiq_queueing].increment( + labels: labels, + success: queue_duration < urgency_requirement + ) + end + def execution_duration_for_urgency(urgency) EXECUTION_URGENCY_DURATIONS.fetch(urgency, DEFAULT_EXECUTION_URGENCY_DURATION) end + + def queueing_duration_for_urgency(urgency) + QUEUEING_URGENCY_DURATIONS.fetch(urgency, DEFAULT_QUEUEING_URGENCY_DURATION) + end end end end diff --git a/lib/gitlab/nav/top_nav_menu_item.rb b/lib/gitlab/nav/top_nav_menu_item.rb index a83cdbe15df..e7790fd77d0 100644 --- a/lib/gitlab/nav/top_nav_menu_item.rb +++ b/lib/gitlab/nav/top_nav_menu_item.rb @@ -21,7 +21,7 @@ module Gitlab href: href, view: view.to_s, css_class: css_class, - data: data || { qa_selector: 'menu_item_link', qa_title: title }, + data: data || { testid: 'menu_item_link', qa_title: title }, partial: partial, component: component } diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb index f7f65c91339..b500df86363 100644 --- a/lib/gitlab/observability.rb +++ b/lib/gitlab/observability.rb @@ -23,7 +23,22 @@ module Gitlab 'https://observe.gitlab.com' end - # Returns true if the Observability feature flag is enabled + def oauth_url + "#{Gitlab::Observability.observability_url}/v1/auth/start" + end + + def tracing_url(project) + "#{Gitlab::Observability.observability_url}/query/#{project.group.id}/#{project.id}/v1/traces" + end + + def provisioning_url(_project) + # TODO Change to correct endpoint when API is ready + Gitlab::Observability.observability_url.to_s + end + + # Returns true if the GitLab Observability UI (GOUI) feature flag is enabled + # + # @deprecated # def enabled?(group = nil) return Feature.enabled?(:observability_group_tab, group) if group @@ -31,6 +46,11 @@ module Gitlab Feature.enabled?(:observability_group_tab) end + # Returns true if Tracing UI is enabled + def tracing_enabled?(project) + Feature.enabled?(:observability_tracing, project) + end + # Returns the embeddable Observability URL of a given URL # # - Validates the URL diff --git a/lib/gitlab/pages/url_builder.rb b/lib/gitlab/pages/url_builder.rb new file mode 100644 index 00000000000..215154b7248 --- /dev/null +++ b/lib/gitlab/pages/url_builder.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +module Gitlab + module Pages + class UrlBuilder + attr_reader :project_namespace + + ALLOWED_ARTIFACT_EXTENSIONS = %w[.html .htm .txt .json .xml .log].freeze + ARTIFACT_URL = "%{host}/-/%{project_path}/-/jobs/%{job_id}/artifacts/%{artifact_path}" + + def initialize(project) + @project = project + @project_namespace, _, @project_path = project.full_path.partition('/') + end + + def pages_url(with_unique_domain: false) + return unique_url if with_unique_domain && unique_domain_enabled? + + project_path_url = "#{config.protocol}://#{project_path}".downcase + + # If the project path is the same as host, we serve it as group page + # On development we ignore the URL port to make it work on GDK + return namespace_url if Rails.env.development? && portless(namespace_url) == project_path_url + # If the project path is the same as host, we serve it as group page + return namespace_url if namespace_url == project_path_url + + "#{namespace_url}/#{project_path}" + end + + def unique_host + return unless unique_domain_enabled? + + URI(unique_url).host + end + + def namespace_pages? + namespace_url == pages_url + end + + def artifact_url(artifact, job) + return unless artifact_url_available?(artifact, job) + + format( + ARTIFACT_URL, + host: namespace_url, + project_path: project_path, + job_id: job.id, + artifact_path: artifact.path) + end + + def artifact_url_available?(artifact, job) + config.enabled && + config.artifacts_server && + ALLOWED_ARTIFACT_EXTENSIONS.include?(File.extname(artifact.name)) && + (config.access_control || job.project.public?) + end + + private + + attr_reader :project, :project_path + + def namespace_url + @namespace_url ||= url_for(project_namespace) + end + + def unique_url + @unique_url ||= url_for(project.project_setting.pages_unique_domain) + end + + def url_for(subdomain) + URI(config.url) + .tap { |url| url.port = config.port } + .tap { |url| url.host.prepend("#{subdomain}.") } + .to_s + .downcase + end + + def portless(url) + URI(url) + .tap { |u| u.port = nil } + .to_s + end + + def unique_domain_enabled? + Feature.enabled?(:pages_unique_domain, project) && + project.project_setting.pages_unique_domain_enabled? + end + + def config + Gitlab.config.pages + end + end + end +end diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb index 8c0f082f61c..422839dcde1 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/query_builder.rb @@ -67,7 +67,7 @@ module Gitlab .select(finder_strategy.final_projections) .where("count <> 0") # filter out the initializer row - model.from(q.arel.as(table_name)) + model.select(Arel.star).from(q.arel.as(table_name)) end private diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index 4f79a3593f4..786ae282c88 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -14,6 +14,7 @@ module Gitlab @finder_query = finder_query @order_by_columns = order_by_columns @table_name = model.table_name + @model = model end def initializer_columns @@ -30,7 +31,11 @@ module Gitlab end def final_projections - ["(#{RECORDS_COLUMN}).*"] + if @model.default_select_columns.is_a?(Array) + @model.default_select_columns.map { |column| "(#{RECORDS_COLUMN}).#{column.name}" } + else + ["(#{RECORDS_COLUMN}).*"] + end end private diff --git a/lib/gitlab/pagination/keyset/order.rb b/lib/gitlab/pagination/keyset/order.rb index 0d8e4ea6fee..a7faef2fdad 100644 --- a/lib/gitlab/pagination/keyset/order.rb +++ b/lib/gitlab/pagination/keyset/order.rb @@ -98,7 +98,7 @@ module Gitlab hash[column_definition.attribute_name] = if field_value.is_a?(Time) # use :inspect formatter to provide specific timezone info # eg 2022-07-05 21:57:56.041499000 +0800 - field_value.to_s(:inspect) + field_value.to_fs(:inspect) elsif field_value.nil? nil elsif lower_named_function?(column_definition) @@ -246,7 +246,8 @@ module Gitlab scopes = where_values.map do |where_value| scope.dup.where(where_value).reorder(self) # rubocop: disable CodeReuse/ActiveRecord end - scope.model.from_union(scopes, remove_duplicates: false, remove_order: false) + + scope.model.select(scope.select_values).from_union(scopes, remove_duplicates: false, remove_order: false) end def to_sql_literal(column_definitions) diff --git a/lib/gitlab/patch/action_cable_redis_listener.rb b/lib/gitlab/patch/action_cable_redis_listener.rb deleted file mode 100644 index b21bee45991..00000000000 --- a/lib/gitlab/patch/action_cable_redis_listener.rb +++ /dev/null @@ -1,26 +0,0 @@ -# frozen_string_literal: true - -# Modifies https://github.com/rails/rails/blob/v6.1.4.6/actioncable/lib/action_cable/subscription_adapter/redis.rb -# so that it is resilient to Redis connection errors. -# See https://github.com/rails/rails/issues/27659. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Patch - module ActionCableRedisListener - private - - def ensure_listener_running - @thread ||= Thread.new do - Thread.current.abort_on_exception = true - - conn = @adapter.redis_connection_for_subscriptions - listen conn - rescue ::Redis::BaseConnectionError - @thread = @raw_client = nil - ::ActionCable.server.restart - end - end - end - end -end diff --git a/lib/gitlab/patch/redis_cache_store.rb b/lib/gitlab/patch/redis_cache_store.rb index 5279c4081b2..041cb2d44bd 100644 --- a/lib/gitlab/patch/redis_cache_store.rb +++ b/lib/gitlab/patch/redis_cache_store.rb @@ -43,7 +43,13 @@ module Gitlab keys = names.map { |name| normalize_key(name, options) } values = failsafe(:patched_read_multi_mget, returning: {}) do - redis.with { |c| pipeline_mget(c, keys) } + redis.with do |c| + if c.is_a?(Gitlab::Redis::MultiStore) + c.with_readonly_pipeline { pipeline_mget(c, keys) } + else + pipeline_mget(c, keys) + end + end end names.zip(values).each_with_object({}) do |(name, value), results| diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 28d195238ea..8a604c7d8a6 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -21,6 +21,9 @@ module Gitlab }, ci: { keys: [:skip, :variable] + }, + integrations: { + keys: [:skip_ci] } }).freeze diff --git a/lib/gitlab/quick_actions/dsl.rb b/lib/gitlab/quick_actions/dsl.rb index dfbc00ef847..1a61b33fd9e 100644 --- a/lib/gitlab/quick_actions/dsl.rb +++ b/lib/gitlab/quick_actions/dsl.rb @@ -61,7 +61,7 @@ module Gitlab # Example: # # explanation do |arguments| - # "Adds label(s) #{arguments.join(' ')}" + # "Adds labels #{arguments.join(' ')}" # end # command :command_key do |arguments| # # Awesome code block @@ -76,7 +76,7 @@ module Gitlab # Example: # # execution_message do |arguments| - # "Added label(s) #{arguments.join(' ')}" + # "Added labels #{arguments.join(' ')}" # end # command :command_key do |arguments| # # Awesome code block diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 96e3112f32f..57ed6c5c35e 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -69,7 +69,7 @@ module Gitlab @updates[:title] = title_param end - desc { _('Add label(s)') } + desc { _('Add labels') } explanation do |labels_param| labels = find_label_references(labels_param) @@ -88,7 +88,7 @@ module Gitlab run_label_command(labels: find_labels(labels_param), command: :label, updates_key: :add_label_ids) end - desc { _('Remove all or specific label(s)') } + desc { _('Remove all or specific labels') } explanation do |labels_param = nil| label_references = labels_param.present? ? find_label_references(labels_param) : [] if label_references.any? @@ -125,7 +125,7 @@ module Gitlab @execution_message[:unlabel] = remove_label_message(label_references) end - desc { _('Replace all label(s)') } + desc { _('Replace all labels') } explanation do |labels_param| labels = find_label_references(labels_param) "Replaces all labels with #{labels.join(' ')} #{'label'.pluralize(labels.count)}." if labels.any? diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index d7e9e1a980b..ae79db723f2 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -259,7 +259,6 @@ module Gitlab current_user.can?(:"set_#{quick_action_target.issue_type}_metadata", quick_action_target) end command :promote_to_incident do - @updates[:issue_type] = :incident @updates[:work_item_type] = ::WorkItems::Type.default_by_type(:incident) end diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index e549ee2e43a..e01be4e0604 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -44,7 +44,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_assignees? - _('Remove all or specific assignee(s)') + _('Remove all or specific assignees') else _('Remove assignee') end diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index c374593bf01..9798b0eca2c 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -202,7 +202,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_reviewers? - _('Assign reviewer(s)') + _('Assign reviewers') else _('Assign reviewer') end @@ -244,7 +244,7 @@ module Gitlab desc do if quick_action_target.allows_multiple_reviewers? - _('Remove all or specific reviewer(s)') + _('Remove all or specific reviewers') else _('Remove reviewer') end diff --git a/lib/gitlab/quick_actions/work_item_actions.rb b/lib/gitlab/quick_actions/work_item_actions.rb index 5664410f3ca..a5c3c6a56be 100644 --- a/lib/gitlab/quick_actions/work_item_actions.rb +++ b/lib/gitlab/quick_actions/work_item_actions.rb @@ -98,7 +98,7 @@ module Gitlab def success_msg { type: _('Type changed successfully.'), - promote_to: _("Work Item promoted successfully.") + promote_to: _("Work item promoted successfully.") } end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index d36ef6b99ee..7f4d611a490 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -6,15 +6,15 @@ module Gitlab include Gitlab::Utils::StrongMemoize class PipelinedDiffError < StandardError - def initialize(result_primary, result_secondary) - @result_primary = result_primary - @result_secondary = result_secondary + def initialize(non_default_store_result, default_store_result) + @non_default_store_result = non_default_store_result + @default_store_result = default_store_result end def message "Pipelined command executed on both stores successfully but results differ between them. " \ - "Result from the primary: #{@result_primary.inspect}. " \ - "Result from the secondary: #{@result_secondary.inspect}." + "Result from the non-default store: #{@non_default_store_result.inspect}. " \ + "Result from the default store: #{@default_store_result.inspect}." end end @@ -24,11 +24,17 @@ module Gitlab end end + class NestedReadonlyPipelineError < StandardError + def message + 'Nested use of with_readonly_pipeline is detected.' + end + end + attr_reader :primary_store, :secondary_store, :instance_name FAILED_TO_READ_ERROR_MESSAGE = 'Failed to read from the redis default_store.' - FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis primary_store.' - FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis primary_store.' + FAILED_TO_WRITE_ERROR_MESSAGE = 'Failed to write to the redis non_default_store.' + FAILED_TO_RUN_PIPELINE = 'Failed to execute pipeline on the redis non_default_store.' SKIP_LOG_METHOD_MISSING_FOR_COMMANDS = %i[info].freeze @@ -100,6 +106,25 @@ module Gitlab validate_stores! end + # Pipelines are sent to both instances by default since + # they could execute both read and write commands. + # + # But for pipelines that only consists of read commands, this method + # can be used to scope the pipeline and send it only to the default store. + def with_readonly_pipeline + raise NestedReadonlyPipelineError if readonly_pipeline? + + Thread.current[:readonly_pipeline] = true + + yield + ensure + Thread.current[:readonly_pipeline] = false + end + + def readonly_pipeline? + Thread.current[:readonly_pipeline].present? + end + # rubocop:disable GitlabSecurity/PublicSend READ_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| @@ -123,7 +148,7 @@ module Gitlab PIPELINED_COMMANDS.each do |name| define_method(name) do |*args, **kwargs, &block| - if use_primary_and_secondary_stores? + if use_primary_and_secondary_stores? && !readonly_pipeline? pipelined_both(name, *args, **kwargs, &block) else send_command(default_store, name, *args, **kwargs, &block) @@ -192,7 +217,7 @@ module Gitlab use_primary_store_as_default? ? primary_store : secondary_store end - def fallback_store + def non_default_store use_primary_store_as_default? ? secondary_store : primary_store end @@ -252,36 +277,39 @@ module Gitlab end def write_both(command_name, *args, **kwargs, &block) + result = send_command(default_store, command_name, *args, **kwargs, &block) + + # write to the non-default store only if write on default store is successful begin - send_command(primary_store, command_name, *args, **kwargs, &block) + send_command(non_default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_WRITE_ERROR_MESSAGE) end - send_command(secondary_store, command_name, *args, **kwargs, &block) + result end # Run the entire pipeline on both stores. We assume that `&block` is idempotent. def pipelined_both(command_name, *args, **kwargs, &block) + result_default = send_command(default_store, command_name, *args, **kwargs, &block) + begin - result_primary = send_command(primary_store, command_name, *args, **kwargs, &block) + result_non_default = send_command(non_default_store, command_name, *args, **kwargs, &block) rescue StandardError => e log_error(e, command_name, multi_store_error_message: FAILED_TO_RUN_PIPELINE) end - result_secondary = send_command(secondary_store, command_name, *args, **kwargs, &block) - # Pipelined commands return an array with all results. If they differ, log an error - if result_primary && result_primary != result_secondary - error = PipelinedDiffError.new(result_primary, result_secondary) + if result_non_default && result_non_default != result_default + error = PipelinedDiffError.new(result_non_default, result_default) error.set_backtrace(Thread.current.backtrace[1..]) # Manually set backtrace, since the error is not `raise`d log_error(error, command_name) increment_pipelined_command_error_count(command_name) end - result_secondary + result_default end def same_redis_store? diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 26ca9d2547c..4e666dbaf77 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -78,6 +78,10 @@ module Gitlab @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o end + def npm_package_name_regex_message + 'should be a valid NPM package name: https://github.com/npm/validate-npm-package-name#naming-rules.' + end + def nuget_package_name_regex @nuget_package_name_regex ||= %r{\A[-+\.\_a-zA-Z0-9]+\z}.freeze end @@ -177,6 +181,10 @@ module Gitlab @semver_regex ||= Regexp.new("\\A#{::Gitlab::Regex.unbounded_semver_regex.source}\\z", ::Gitlab::Regex.unbounded_semver_regex.options).freeze end + def semver_regex_message + 'should follow SemVer: https://semver.org' + end + # These partial semver regexes are intended for use in composing other # regexes rather than being used alone. def _semver_major_minor_patch_regex diff --git a/lib/gitlab/search/found_blob.rb b/lib/gitlab/search/found_blob.rb index 79d6cfc84a3..c9051b6a5ff 100644 --- a/lib/gitlab/search/found_blob.rb +++ b/lib/gitlab/search/found_blob.rb @@ -9,7 +9,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize include BlobActiveModel - attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count + attr_reader :project, :content_match, :blob_path, :highlight_line, :matched_lines_count, :group_level_blob, :group PATH_REGEXP = /\A(?<ref>[^:]*):(?<path>[^\x00]*)\x00/.freeze CONTENT_REGEXP = /^(?<ref>[^:]*):(?<path>[^\x00]*)\x00(?<startline>\d+)\x00/.freeze @@ -31,14 +31,17 @@ module Gitlab @binary_data = opts.fetch(:data, nil) @per_page = opts.fetch(:per_page, 20) @project = opts.fetch(:project, nil) + @group = opts.fetch(:group, nil) # Some callers (e.g. Elasticsearch) do not have the Project object, # yet they can trigger many calls in one go, # causing duplicated queries. # Allow those to just pass project_id instead. @project_id = opts.fetch(:project_id, nil) + @group_id = opts.fetch(:group_id, nil) @content_match = opts.fetch(:content_match, nil) @blob_path = opts.fetch(:blob_path, nil) @repository = opts.fetch(:repository, nil) + @group_level_blob = opts.fetch(:group_level_blob, false) end def id diff --git a/lib/gitlab/search/found_wiki_page.rb b/lib/gitlab/search/found_wiki_page.rb index 99ca6a79fe2..650bae2af4d 100644 --- a/lib/gitlab/search/found_wiki_page.rb +++ b/lib/gitlab/search/found_wiki_page.rb @@ -14,7 +14,8 @@ module Gitlab # @param found_blob [Gitlab::Search::FoundBlob] def initialize(found_blob) super - @wiki = found_blob.project.wiki + + @wiki ||= found_blob.project.wiki end def to_ability_name @@ -23,3 +24,5 @@ module Gitlab end end end + +Gitlab::Search::FoundWikiPage.prepend_mod_with('Gitlab::Search::FoundWikiPage') diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index a733dca6a56..4fedc450f9b 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -107,11 +107,7 @@ module Gitlab def users return User.none unless Ability.allowed?(current_user, :read_users_list) - if Feature.enabled?(:autocomplete_users_use_search_service) - UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute - else - UsersFinder.new(current_user, search: query).execute - end + UsersFinder.new(current_user, { search: query, use_minimum_char_limit: false }).execute end # highlighting is only performed by Elasticsearch backed results diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index ec514adafc8..d5d9b794cd9 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -126,12 +126,12 @@ module Gitlab end def self.without_statement_timeout - Gitlab::Database::EachDatabase.each_database_connection do |connection| + Gitlab::Database::EachDatabase.each_connection do |connection| connection.execute('SET statement_timeout=0') end yield ensure - Gitlab::Database::EachDatabase.each_database_connection do |connection| + Gitlab::Database::EachDatabase.each_connection do |connection| connection.execute('RESET statement_timeout') end end diff --git a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb index c77db02061c..2cd9afc5bdc 100644 --- a/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb +++ b/lib/gitlab/seeders/ci/runner/runner_fleet_pipeline_seeder.rb @@ -169,7 +169,10 @@ module Gitlab } logger.info(message: 'Creating build', **build_attrs) - ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + ::Ci::Build.transaction do + build = ::Ci::Build.new(importing: true, **build_attrs).tap(&:save!) + ::Ci::RunningBuild.upsert_shared_runner_build!(build) if build.running? && build.shared_runner_build? + end end def random_pipeline_status diff --git a/lib/gitlab/sidekiq_logging/structured_logger.rb b/lib/gitlab/sidekiq_logging/structured_logger.rb index c4566a6dc2a..56762c0fb4b 100644 --- a/lib/gitlab/sidekiq_logging/structured_logger.rb +++ b/lib/gitlab/sidekiq_logging/structured_logger.rb @@ -78,6 +78,8 @@ module Gitlab job_status = if job_exception 'fail' + elsif job['dropped'] + 'dropped' elsif job['deferred'] 'deferred' else @@ -87,12 +89,19 @@ module Gitlab payload['message'] = "#{message}: #{job_status}: #{payload['duration_s']} sec" payload['job_status'] = job_status payload['job_deferred_by'] = job['deferred_by'] if job['deferred'] + payload['deferred_count'] = job['deferred_count'] if job['deferred'] Gitlab::ExceptionLogFormatter.format!(job_exception, payload) if job_exception db_duration = ActiveRecord::LogSubscriber.runtime payload['db_duration_s'] = Gitlab::Utils.ms_to_round_sec(db_duration) + job_urgency = payload['class'].safe_constantize&.get_urgency.to_s + unless job_urgency.empty? + payload['urgency'] = job_urgency + payload['target_duration_s'] = Gitlab::Metrics::SidekiqSlis.execution_duration_for_urgency(job_urgency) + end + payload end diff --git a/lib/gitlab/sidekiq_middleware.rb b/lib/gitlab/sidekiq_middleware.rb index ec2a6472809..614cd11421e 100644 --- a/lib/gitlab/sidekiq_middleware.rb +++ b/lib/gitlab/sidekiq_middleware.rb @@ -7,7 +7,7 @@ module Gitlab # The result of this method should be passed to # Sidekiq's `config.server_middleware` method # eg: `config.server_middleware(&Gitlab::SidekiqMiddleware.server_configurator)` - def self.server_configurator(metrics: true, arguments_logger: true, defer_jobs: true) + def self.server_configurator(metrics: true, arguments_logger: true, skip_jobs: true) lambda do |chain| # Size limiter should be placed at the top chain.add ::Gitlab::SidekiqMiddleware::SizeLimiter::Server @@ -40,7 +40,7 @@ module Gitlab # so we can compare the latest WAL location against replica chain.add ::Gitlab::SidekiqMiddleware::DuplicateJobs::Server chain.add ::Gitlab::Database::LoadBalancing::SidekiqServerMiddleware - chain.add ::Gitlab::SidekiqMiddleware::DeferJobs if defer_jobs + chain.add ::Gitlab::SidekiqMiddleware::SkipJobs if skip_jobs end end diff --git a/lib/gitlab/sidekiq_middleware/defer_jobs.rb b/lib/gitlab/sidekiq_middleware/defer_jobs.rb deleted file mode 100644 index 0a12667865c..00000000000 --- a/lib/gitlab/sidekiq_middleware/defer_jobs.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module SidekiqMiddleware - class DeferJobs - DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) - FEATURE_FLAG_PREFIX = "defer_sidekiq_jobs" - - DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) - - # There are 2 scenarios under which this middleware defers a job - # 1. defer_sidekiq_jobs_#{worker_name} FF, jobs are deferred indefinitely until this feature flag - # is turned off or when Feature.enabled? returns false by chance while using `percentage of time` value. - # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator - # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). - def call(worker, job, _queue) - # ActiveJobs have wrapped class stored in 'wrapped' key - resolved_class = job['wrapped']&.safe_constantize || worker.class - defer_job, delay, deferred_by = defer_job_info(resolved_class, job) - - if !!defer_job - # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' - job['deferred'] = true - job['deferred_by'] = deferred_by - - worker.class.perform_in(delay, *job['args']) - counter.increment({ worker: worker.class.name }) - - # This breaks the middleware chain and return - return - end - - yield - end - - private - - def defer_job_info(worker_class, job) - if defer_job_by_ff?(worker_class) - [true, DELAY, :feature_flag] - elsif defer_job_by_database_health_signal?(job, worker_class) - [true, worker_class.database_health_check_attrs[:delay_by], :database_health_check] - end - end - - def defer_job_by_ff?(worker_class) - Feature.enabled?( - :"#{FEATURE_FLAG_PREFIX}_#{worker_class.name}", - type: :worker, - default_enabled_if_undefined: false - ) - end - - def defer_job_by_database_health_signal?(job, worker_class) - unless worker_class.respond_to?(:defer_on_database_health_signal?) && - worker_class.defer_on_database_health_signal? - return false - end - - health_check_attrs = worker_class.database_health_check_attrs - job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first - - health_context = Gitlab::Database::HealthStatus::Context.new( - DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), - job_base_model.connection, - health_check_attrs[:gitlab_schema], - health_check_attrs[:tables] - ) - - Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) - end - - def counter - @counter ||= Gitlab::Metrics.counter(:sidekiq_jobs_deferred_total, 'The number of jobs deferred') - end - end - end -end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index 3ed9c1743ed..46939d70c9e 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -97,13 +97,13 @@ module Gitlab local connection = ARGV[i] local current_offset = cookie.offsets[connection] local new_offset = tonumber(ARGV[i+1]) - if not current_offset or current_offset < new_offset then + if not current_offset or (new_offset and current_offset < new_offset) then cookie.offsets[connection] = new_offset cookie.wal_locations[connection] = ARGV[i+2] end end - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "keepttl") LUA def latest_wal_locations @@ -147,10 +147,7 @@ module Gitlab end local cookie = cmsgpack.unpack(cookie_msgpack) cookie.deduplicated = "1" - local ttl = redis.call("ttl", KEYS[1]) - if ttl > 0 then - redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", ttl) - end + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "keepttl") LUA def should_reschedule? diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index b3c3c94a0a3..058c23178f8 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -18,7 +18,7 @@ module Gitlab SIDEKIQ_QUEUE_DURATION_BUCKETS = [10, 60].freeze # These labels from Gitlab::SidekiqMiddleware::MetricsHelper are included in SLI metrics - SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency].freeze + SIDEKIQ_SLI_LABELS = [:worker, :feature_category, :urgency, :external_dependencies].freeze class << self include ::Gitlab::SidekiqMiddleware::MetricsHelper @@ -64,7 +64,8 @@ module Gitlab end end - Gitlab::Metrics::SidekiqSlis.initialize_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) + Gitlab::Metrics::SidekiqSlis.initialize_execution_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_execution_application_slis) + Gitlab::Metrics::SidekiqSlis.initialize_queueing_slis!(possible_sli_labels) if ::Feature.enabled?(:sidekiq_queueing_application_slis) end end @@ -147,6 +148,11 @@ module Gitlab Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) end + + if ::Feature.enabled?(:sidekiq_queueing_application_slis) + sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + end end end diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb index acc3e1712ab..b19cc994d32 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/validator.rb @@ -33,7 +33,8 @@ module Gitlab EXEMPT_WORKER_NAMES = %w[BackgroundMigrationWorker BackgroundMigration::CiDatabaseWorker Database::BatchedBackgroundMigrationWorker - Database::BatchedBackgroundMigration::CiDatabaseWorker].to_set + Database::BatchedBackgroundMigration::CiDatabaseWorker + RedisMigrationWorker].to_set JOB_STATUS_KEY = 'size_limiter' diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb new file mode 100644 index 00000000000..8932607df52 --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -0,0 +1,125 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + class SkipJobs + DELAY = ENV.fetch("SIDEKIQ_DEFER_JOBS_DELAY", 5.minutes) + RUN_FEATURE_FLAG_PREFIX = "run_sidekiq_jobs" + DROP_FEATURE_FLAG_PREFIX = "drop_sidekiq_jobs" + + DatabaseHealthStatusChecker = Struct.new(:id, :job_class_name) + + COUNTER = :sidekiq_jobs_skipped_total + + def initialize + @metrics = init_metrics + end + + # This middleware decides whether a job is dropped, deferred or runs normally. + # In short: + # - `drop_sidekiq_jobs_#{worker_name}` FF enabled (disabled by default) --> drops the job + # - `run_sidekiq_jobs_#{worker_name}` FF disabled (enabled by default) --> defers the job + # + # DROPPING JOBS + # A job is dropped when `drop_sidekiq_jobs_#{worker_name}` FF is enabled. This FF is disabled by default for + # all workers. Dropped jobs are completely ignored and not requeued for future processing. + # + # DEFERRING JOBS + # Deferred jobs are rescheduled to perform in the future. + # There are 2 scenarios under which this middleware defers a job: + # 1. When run_sidekiq_jobs_#{worker_name} FF is disabled. This FF is enabled by default + # for all workers. + # 2. Gitlab::Database::HealthStatus, on evaluating the db health status if it returns any indicator + # with stop signal, the jobs will be delayed by 'x' seconds (set in worker). + # + # Dropping jobs takes higher priority over deferring jobs. For example, when `drop_sidekiq_jobs` is enabled and + # `run_sidekiq_jobs` is disabled, it results to jobs being dropped. + def call(worker, job, _queue) + # ActiveJobs have wrapped class stored in 'wrapped' key + resolved_class = job['wrapped']&.safe_constantize || worker.class + if drop_job?(resolved_class) + # no-op, drop the job entirely + drop_job!(job, worker) + return + elsif !!defer_job?(resolved_class, job) + defer_job!(job, worker) + return + end + + yield + end + + private + + def defer_job?(worker_class, job) + if !run_job_by_ff?(worker_class) + @delay = DELAY + @deferred_by = :feature_flag + true + elsif defer_job_by_database_health_signal?(job, worker_class) + @delay = worker_class.database_health_check_attrs[:delay_by] + @deferred_by = :database_health_check + true + end + end + + def run_job_by_ff?(worker_class) + # always returns true by default for all workers unless the FF is specifically disabled, e.g. during an incident + Feature.enabled?( + :"#{RUN_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: true + ) + end + + def defer_job_by_database_health_signal?(job, worker_class) + unless worker_class.respond_to?(:defer_on_database_health_signal?) && + worker_class.defer_on_database_health_signal? + return false + end + + health_check_attrs = worker_class.database_health_check_attrs + job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + health_context = Gitlab::Database::HealthStatus::Context.new( + DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), + job_base_model.connection, + health_check_attrs[:gitlab_schema], + health_check_attrs[:tables] + ) + + Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) + end + + def drop_job?(worker_class) + Feature.enabled?( + :"#{DROP_FEATURE_FLAG_PREFIX}_#{worker_class.name}", + type: :worker, + default_enabled_if_undefined: false + ) + end + + def drop_job!(job, worker) + job['dropped'] = true + @metrics.fetch(COUNTER).increment({ worker: worker.class.name, action: "dropped" }) + end + + def defer_job!(job, worker) + # Referred in job_logger's 'log_job_done' method to compute proper 'job_status' + job['deferred'] = true + job['deferred_by'] = @deferred_by + job['deferred_count'] ||= 0 + job['deferred_count'] += 1 + + worker.class.perform_in(@delay, *job['args']) + @metrics.fetch(COUNTER).increment({ worker: worker.class.name, action: "deferred" }) + end + + def init_metrics + { + COUNTER => Gitlab::Metrics.counter(COUNTER, 'The number of skipped jobs') + } + end + end + end +end diff --git a/lib/gitlab/signed_commit.rb b/lib/gitlab/signed_commit.rb index 410e71f51a1..be6592dd231 100644 --- a/lib/gitlab/signed_commit.rb +++ b/lib/gitlab/signed_commit.rb @@ -34,13 +34,19 @@ module Gitlab def signature_text strong_memoize(:signature_text) do - @signature_data.itself ? @signature_data[0] : nil + @signature_data.itself ? @signature_data[:signature] : nil end end def signed_text strong_memoize(:signed_text) do - @signature_data.itself ? @signature_data[1] : nil + @signature_data.itself ? @signature_data[:signed_text] : nil + end + end + + def signer + strong_memoize(:signer) do + @signature_data.itself ? @signature_data[:signer] : nil end end diff --git a/lib/gitlab/slash_commands/presenters/access.rb b/lib/gitlab/slash_commands/presenters/access.rb index c9c5c6da3bf..e098762f290 100644 --- a/lib/gitlab/slash_commands/presenters/access.rb +++ b/lib/gitlab/slash_commands/presenters/access.rb @@ -21,8 +21,8 @@ module Gitlab def deactivated ephemeral_response(text: <<~MESSAGE) - You are not allowed to perform the given chatops command since - your account has been deactivated by your administrator. + You are not allowed to perform the given ChatOps command. Most likely + your #{Gitlab.config.gitlab.url} account needs to be reactivated. Please log back in from a web browser to reactivate your account at #{Gitlab.config.gitlab.url} MESSAGE diff --git a/lib/gitlab/spamcheck/client.rb b/lib/gitlab/spamcheck/client.rb index 0afaf46fa9b..d13c3be0a09 100644 --- a/lib/gitlab/spamcheck/client.rb +++ b/lib/gitlab/spamcheck/client.rb @@ -58,7 +58,7 @@ module Gitlab pb.title = spammable.spam_title || '' if pb.respond_to?(:title) pb.description = spammable.spam_description || '' if pb.respond_to?(:description) pb.text = spammable.spammable_text || '' if pb.respond_to?(:text) - pb.type = spammable.spammable_entity_type if pb.respond_to?(:type) + pb.type = spammable.to_ability_name if pb.respond_to?(:type) pb.created_at = convert_to_pb_timestamp(spammable.created_at) if spammable.created_at pb.updated_at = convert_to_pb_timestamp(spammable.updated_at) if spammable.updated_at pb.action = ACTION_MAPPING.fetch(context.fetch(:action)) if context.has_key?(:action) diff --git a/lib/gitlab/ssh/commit.rb b/lib/gitlab/ssh/commit.rb index d9ac8c1b881..7d7cc529b1a 100644 --- a/lib/gitlab/ssh/commit.rb +++ b/lib/gitlab/ssh/commit.rb @@ -10,7 +10,7 @@ module Gitlab end def attributes - signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, @commit.committer_email) + signature = ::Gitlab::Ssh::Signature.new(signature_text, signed_text, signer, @commit.committer_email) { commit_sha: @commit.sha, diff --git a/lib/gitlab/ssh/signature.rb b/lib/gitlab/ssh/signature.rb index 763d89116f1..6b0cab75557 100644 --- a/lib/gitlab/ssh/signature.rb +++ b/lib/gitlab/ssh/signature.rb @@ -11,15 +11,17 @@ module Gitlab GIT_NAMESPACE = 'git' - def initialize(signature_text, signed_text, committer_email) + def initialize(signature_text, signed_text, signer, committer_email) @signature_text = signature_text @signed_text = signed_text + @signer = signer @committer_email = committer_email end def verification_status strong_memoize(:verification_status) do next :unverified unless all_attributes_present? + next :verified_system if verified_by_gitlab? next :unverified unless valid_signature_blob? next :unknown_key unless signed_by_key next :other_user unless committer @@ -81,6 +83,15 @@ module Gitlab nil end end + + # If a commit is signed by Gitaly, the Gitaly returns `SIGNER_SYSTEM` as a signer + # In order to calculate it, the signature is Verified using the Gitaly's public key: + # https://gitlab.com/gitlab-org/gitaly/-/blob/v16.2.0-rc2/internal/gitaly/service/commit/commit_signatures.go#L63 + # + # It is safe to skip verification step if the commit has been signed by Gitaly + def verified_by_gitlab? + @signer == :SIGNER_SYSTEM + end end end end diff --git a/lib/gitlab/subscription_portal.rb b/lib/gitlab/subscription_portal.rb index 1d9ecb624b2..bbcefabcb40 100644 --- a/lib/gitlab/subscription_portal.rb +++ b/lib/gitlab/subscription_portal.rb @@ -21,6 +21,14 @@ module Gitlab def self.renewal_service_email 'renewals-service@customers.gitlab.com' end + + def self.default_staging_customer_portal_url + 'https://customers.staging.gitlab.com' + end + + def self.default_production_customer_portal_url + 'https://customers.gitlab.com' + end end end diff --git a/lib/gitlab/task_helpers.rb b/lib/gitlab/task_helpers.rb index b9800a4db73..f756d229ba1 100644 --- a/lib/gitlab/task_helpers.rb +++ b/lib/gitlab/task_helpers.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true require 'rainbow/ext/string' -require_relative 'utils/strong_memoize' +require 'gitlab/utils/all' # rubocop:disable Rails/Output module Gitlab diff --git a/lib/gitlab/testing/action_cable_blocker.rb b/lib/gitlab/testing/action_cable_blocker.rb new file mode 100644 index 00000000000..aebb0732035 --- /dev/null +++ b/lib/gitlab/testing/action_cable_blocker.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +# rubocop:disable Style/ClassVars + +# This is inspired by http://www.salsify.com/blog/engineering/tearing-capybara-ajax-tests +# Rack middleware that keeps track of the number of active requests and can block new requests. +module Gitlab + module Testing + class ActionCableBlocker + @@num_active_requests = Concurrent::AtomicFixnum.new(0) + @@block_requests = Concurrent::AtomicBoolean.new(false) + + # Returns the number of requests the server is currently processing. + def self.num_active_requests + @@num_active_requests.value + end + + # Prevents the server from accepting new requests. Any new requests will be skipped. + def self.block_requests! + @@block_requests.value = true + end + + # Allows the server to accept requests again. + def self.allow_requests! + @@block_requests.value = false + end + + def self.install + ::ActionCable::Server::Worker.set_callback :work, :around do |_, inner| + @@num_active_requests.increment + + inner.call if @@block_requests.false? + ensure + @@num_active_requests.decrement + end + end + end + end +end +# rubocop:enable Style/ClassVars diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 065ede75c60..bd42586731e 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -22,6 +22,10 @@ module Gitlab key_path end + def events + events_from_new_structure || events_from_old_structure || {} + end + def to_h attributes end @@ -44,7 +48,7 @@ module Gitlab def validate! unless skip_validation? - self.class.schemer.validate(attributes.stringify_keys).each do |error| + self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| error_message = <<~ERROR_MSG Error type: #{error['type']} Data: #{error['data']} @@ -102,6 +106,19 @@ module Gitlab @metrics_yaml ||= definitions.values.map(&:to_h).map(&:deep_stringify_keys).to_yaml end + def metric_definitions_changed? + return false unless Rails.env.development? + + return false if @last_change_check && @last_change_check > 3.seconds.ago + + @last_change_check = Time.current + + last_change = Dir.glob(paths).map { |f| File.mtime(f) }.max + did_change = @last_metric_update != last_change + @last_metric_update = last_change + did_change + end + private def load_all! @@ -146,6 +163,20 @@ module Gitlab def skip_validation? !!attributes[:skip_validation] || @skip_validation || attributes[:status] == SKIP_VALIDATION_STATUS end + + def events_from_new_structure + events = attributes[:events] + return unless events + + events.to_h { |event| [event[:name], event[:unique].to_sym] } + end + + def events_from_old_structure + options_events = attributes.dig(:options, :events) + return unless options_events + + options_events.index_with { nil } + end end end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index 4b38809dde4..0edd9f7914a 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -15,10 +15,14 @@ module Gitlab DATABASE_SOURCE = 'database' REDIS_SOURCE = 'redis_hll' + INTERNAL_EVENTS_SOURCE = 'internal_events' SOURCES = { DATABASE_SOURCE => Sources::PostgresHll, - REDIS_SOURCE => Sources::RedisHll + REDIS_SOURCE => Sources::RedisHll, + # Same strategy as RedisHLL, since they are a part of internal events + # and should get counted together with other RedisHLL-based aggregations + INTERNAL_EVENTS_SOURCE => Sources::RedisHll }.freeze end end diff --git a/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb new file mode 100644 index 00000000000..f5529b96678 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/batched_background_migrations_metric.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class BatchedBackgroundMigrationsMetric < DatabaseMetric + relation { Gitlab::Database::BackgroundMigration::BatchedMigration.with_status(:finished) } + + timestamp_column(:finished_at) + + operation :count + + def value + relation.map do |batched_migration| + { + job_class_name: batched_migration.job_class_name, + elapsed_time: batched_migration.finished_at.to_i - batched_migration.started_at.to_i + } + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb new file mode 100644 index 00000000000..25a45a259e2 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_projects_with_jira_dvcs_integration_metric.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountProjectsWithJiraDvcsIntegrationMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + raise ArgumentError, "option 'cloud' must be a boolean" unless [true, false].include?(options[:cloud]) + end + + relation do |options| + ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: options[:cloud]) + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb new file mode 100644 index 00000000000..0a796c9fae9 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_gbp_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSlackAppInstallationsGbpMetric < DatabaseMetric + operation :count + + relation { SlackIntegration.with_bot } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb new file mode 100644 index 00000000000..af9cf957dab --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_slack_app_installations_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountSlackAppInstallationsMetric < DatabaseMetric + operation :count + + relation { SlackIntegration } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb index 7c646281598..d57dd7eac20 100644 --- a/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/generic_metric.rb @@ -32,9 +32,9 @@ module Gitlab super(metric_definition.reverse_merge(time_frame: 'none')) end - def value + def value(...) alt_usage_data(fallback: self.class.fallback) do - self.class.metric_value.call + self.class.metric_value.call(...) end end diff --git a/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb b/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb new file mode 100644 index 00000000000..ae1d076af19 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/gitaly_apdex_metric.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class GitalyApdexMetric < PrometheusMetric + value do |client| + result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + + break FALLBACK unless result + + result['value'].last.to_f + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb index 409027925d1..2ce7e95ce77 100644 --- a/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/index_inconsistencies_metric.rb @@ -6,7 +6,7 @@ module Gitlab module Instrumentations class IndexInconsistenciesMetric < GenericMetric value do - runner = Gitlab::Database::SchemaValidation::Runner.new(structure_sql, database, validators: validators) + runner = Gitlab::Schema::Validation::Runner.new(structure_sql, database, validators: validators) inconsistencies = runner.execute @@ -23,19 +23,19 @@ module Gitlab def database database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] - Gitlab::Database::SchemaValidation::Database.new(database_model.connection) + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) end def structure_sql stucture_sql_path = Rails.root.join('db/structure.sql') - Gitlab::Database::SchemaValidation::StructureSql.new(stucture_sql_path) + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) end def validators [ - Gitlab::Database::SchemaValidation::Validators::MissingIndexes, - Gitlab::Database::SchemaValidation::Validators::DifferentDefinitionIndexes, - Gitlab::Database::SchemaValidation::Validators::ExtraIndexes + Gitlab::Schema::Validation::Validators::MissingIndexes, + Gitlab::Schema::Validation::Validators::DifferentDefinitionIndexes, + Gitlab::Schema::Validation::Validators::ExtraIndexes ] end end diff --git a/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb new file mode 100644 index 00000000000..7667cff06e0 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/ldap_encrypted_secrets_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class LdapEncryptedSecretsMetric < GenericMetric + value do + Gitlab::Auth::Ldap::Config.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb b/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb new file mode 100644 index 00000000000..9bfe12d8ead --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/operating_system_metric.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class OperatingSystemMetric < GenericMetric + value do + ohai_data = Ohai::System.new.tap do |oh| + oh.all_plugins(['platform']) + end.data + + platform = ohai_data['platform'] + if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') + platform = 'raspbian' + end + + "#{platform}-#{ohai_data['platform_version']}" + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb new file mode 100644 index 00000000000..ab1298b63c3 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/prometheus_metric.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class PrometheusMetric < GenericMetric + # Usage example + # + # class GitalyApdexMetric < PrometheusMetric + # value do + # result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first + # + # break FALLBACK unless result + # + # result['value'].last.to_f + # end + # end + def value + with_prometheus_client(verify: false, fallback: FALLBACK) do |client| + super(client) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb new file mode 100644 index 00000000000..a481f7a5682 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/schema_inconsistencies_metric.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SchemaInconsistenciesMetric < GenericMetric + MAX_INCONSISTENCIES = 150 # Limit the number of inconsistencies reported to avoid large payloads + + value do + runner = Gitlab::Schema::Validation::Runner.new(structure_sql, database, validators: validators) + + inconsistencies = runner.execute + + inconsistencies.take(MAX_INCONSISTENCIES).map do |inconsistency| + { + object_name: inconsistency.object_name, + inconsistency_type: inconsistency.type, + object_type: inconsistency.object_type + } + end + end + + class << self + private + + def validators + Gitlab::Schema::Validation::Validators::Base.all_validators + end + + def database + database_model = Gitlab::Database.database_base_models[Gitlab::Database::MAIN_DATABASE_NAME] + Gitlab::Schema::Validation::Sources::Database.new(database_model.connection) + end + + def structure_sql + stucture_sql_path = Rails.root.join('db/structure.sql') + Gitlab::Schema::Validation::Sources::StructureSql.new(stucture_sql_path) + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb b/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb new file mode 100644 index 00000000000..1e1925f9933 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/smtp_encrypted_secrets_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class SmtpEncryptedSecretsMetric < GenericMetric + value do + Gitlab::Email::SmtpConfig.encrypted_secrets.active? + end + end + end + end + end +end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 72168bce782..ab041a31bde 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -152,22 +152,6 @@ module Gitlab } end - def system_usage_data_settings - { - settings: { - ldap_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Auth::Ldap::Config.encrypted_secrets.active? }, - smtp_encrypted_secrets_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::SmtpConfig.encrypted_secrets.active? }, - operating_system: alt_usage_data(fallback: nil) { operating_system }, - gitaly_apdex: alt_usage_data { gitaly_apdex }, - collected_data_categories: add_metric('CollectedDataCategoriesMetric', time_frame: 'none'), - service_ping_features_enabled: add_metric('ServicePingFeaturesMetric', time_frame: 'none'), - snowplow_enabled: add_metric('SnowplowEnabledMetric', time_frame: 'none'), - snowplow_configured_to_gitlab_collector: add_metric('SnowplowConfiguredToGitlabCollectorMetric', time_frame: 'none'), - certificate_based_clusters_ff: add_metric('CertBasedClustersFfMetric') - } - } - end - def system_usage_data_weekly { counts_weekly: {} @@ -286,16 +270,9 @@ module Gitlab response[:"instances_#{name}_active"] = count(Integration.active.where(instance: true, type: type)) response[:"projects_inheriting_#{name}_active"] = count(Integration.active.where.not(project: nil).where.not(inherit_from_id: nil).where(type: type)) response[:"groups_inheriting_#{name}_active"] = count(Integration.active.where.not(group: nil).where.not(inherit_from_id: nil).where(type: type)) - end.merge(jira_usage, jira_import_usage) + end.merge(jira_import_usage) # rubocop: enable UsageData/LargeTable: end - - def jira_usage - { - projects_jira_dvcs_cloud_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled), - projects_jira_dvcs_server_active: count(ProjectFeatureUsage.with_jira_dvcs_integration_enabled(cloud: false)) - } - end # rubocop: enable CodeReuse/ActiveRecord def jira_import_usage @@ -328,17 +305,6 @@ module Gitlab } end - def operating_system - ohai_data = Ohai::System.new.tap do |oh| - oh.all_plugins(['platform']) - end.data - - platform = ohai_data['platform'] - platform = 'raspbian' if ohai_data['platform'] == 'debian' && ohai_data['kernel']['machine']&.include?('armv') - - "#{platform}-#{ohai_data['platform_version']}" - end - # Source: https://gitlab.com/gitlab-data/analytics/blob/master/transform/snowflake-dbt/data/ping_metrics_to_stage_mapping_data.csv def usage_activity_by_stage(key = :usage_activity_by_stage, time_period = {}) { @@ -371,7 +337,11 @@ module Gitlab group_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.group_type, time_period), group_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.group_type, time_period), project_clusters_disabled: clusters_user_distinct_count(::Clusters::Cluster.disabled.project_type, time_period), - project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period) + project_clusters_enabled: clusters_user_distinct_count(::Clusters::Cluster.enabled.project_type, time_period), + # These two `projects_slack_x` metrics are owned by the Manage stage, but are in this method as their key paths can't change. + # See https://gitlab.com/gitlab-org/gitlab/-/merge_requests/123442#note_1427961339. + projects_slack_notifications_active: distinct_count(::Project.with_slack_integration.where(time_period), :creator_id), + projects_slack_slash_active: distinct_count(::Project.with_slack_slash_commands_integration.where(time_period), :creator_id) } end # rubocop: enable UsageData/LargeTable @@ -527,7 +497,6 @@ module Gitlab def usage_data_metrics system_usage_data_license - .merge(system_usage_data_settings) .merge(system_usage_data) .merge(system_usage_data_monthly) .merge(system_usage_data_weekly) @@ -543,16 +512,6 @@ module Gitlab time_period.present? ? '28d' : 'none' end - def gitaly_apdex - with_prometheus_client(verify: false, fallback: FALLBACK) do |client| - result = client.query('avg_over_time(gitlab_usage_ping:gitaly_apdex:ratio_avg_over_time_5m[1w])').first - - break FALLBACK unless result - - result['value'].last.to_f - end - end - def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = minimum_id(User) project_creator_id_finish = maximum_id(User) diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index eaa4bf15fe1..e71061c4522 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -15,14 +15,7 @@ module Gitlab # Track event on entity_id # Increment a Redis HLL counter for unique event_name and entity_id # - # All events should be added to known_events yml files lib/gitlab/usage_data_counters/known_events/ - # - # Event example: - # - # - name: g_compliance_dashboard # Unique event name - # # Usage: - # # * Track event: Gitlab::UsageDataCounters::HLLRedisCounter.track_event('g_compliance_dashboard', values: user_id) # * Get unique counts per user: Gitlab::UsageDataCounters::HLLRedisCounter.unique_events(event_names: 'g_compliance_dashboard', start_date: 28.days.ago, end_date: Date.current) class << self @@ -119,8 +112,18 @@ module Gitlab end def load_events(wildcard) - Dir[wildcard].each_with_object([]) do |path, events| - events.push(*load_yaml_from_path(path)) + if Feature.enabled?(:use_metric_definitions_for_events_list) + events = Gitlab::Usage::MetricDefinition.not_removed.values.map do |d| + d.attributes[:options] && d.attributes[:options][:events] + end.flatten.compact.uniq + + events.map do |e| + { name: e }.with_indifferent_access + end + else + Dir[wildcard].each_with_object([]) do |path, events| + events.push(*load_yaml_from_path(path)) + end end end @@ -129,7 +132,7 @@ module Gitlab end def known_events_names - known_events.map { |event| event[:name] } + @known_events_names ||= known_events.map { |event| event[:name] } end def event_for(event_name) diff --git a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb index 31f090e0f51..54464b63fce 100644 --- a/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/issue_activity_unique_counter.rb @@ -38,8 +38,7 @@ module Gitlab class << self def track_issue_created_action(author:, namespace:) - track_snowplow_action(ISSUE_CREATED, author, namespace) - track_unique_action(ISSUE_CREATED, author) + track_internal_action(ISSUE_CREATED, author, namespace) end def track_issue_title_changed_action(author:, project:) @@ -180,14 +179,7 @@ module Gitlab private def track_snowplow_action(event_name, author, container) - namespace, project = case container - when Project - [container.namespace, container] - when Namespaces::ProjectNamespace - [container.parent, container.project] - else - [container, nil] - end + namespace, project = get_params_from_container(container) return unless author @@ -208,6 +200,30 @@ module Gitlab Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: author.id) end + + def track_internal_action(event_name, author, container) + return unless author + + namespace, project = get_params_from_container(container) + + Gitlab::InternalEvents.track_event( + event_name, + user: author, + project: project, + namespace: namespace + ) + end + + def get_params_from_container(container) + case container + when Project + [container.namespace, container] + when Namespaces::ProjectNamespace + [container.parent, container.project] + else + [container, nil] + end + end end end end diff --git a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml index b3d1c51c0e7..fe779a9a25f 100644 --- a/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml +++ b/lib/gitlab/usage_data_counters/known_events/kubernetes_agent.yml @@ -1,2 +1,22 @@ - name: agent_users_using_ci_tunnel aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_ci_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_ci_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_agents_via_ci_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_agents_via_ci_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_users_via_user_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_users_via_user_access + aggregation: monthly +- name: k8s_api_proxy_requests_unique_agents_via_user_access + aggregation: weekly +- name: k8s_api_proxy_requests_unique_agents_via_user_access + aggregation: monthly +- name: flux_git_push_notified_unique_projects + aggregation: weekly +- name: flux_git_push_notified_unique_projects + aggregation: monthly diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index ece2ffea83b..9e8c207a19a 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -4,7 +4,13 @@ module Gitlab module UsageDataCounters class KubernetesAgentCounter < BaseCounter PREFIX = 'kubernetes_agent' - KNOWN_EVENTS = %w[gitops_sync k8s_api_proxy_request flux_git_push_notifications_total].freeze + KNOWN_EVENTS = %w[ + gitops_sync + k8s_api_proxy_request + flux_git_push_notifications_total + k8s_api_proxy_requests_via_ci_access + k8s_api_proxy_requests_via_user_access + ].freeze class << self def increment_event_counts(events) diff --git a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb index 1ed2e891a1f..d26b7ce951d 100644 --- a/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/merge_request_activity_unique_counter.rb @@ -70,9 +70,9 @@ module Gitlab Gitlab::InternalEvents.track_event( MR_USER_CREATE_ACTION, - user_id: user.id, - project_id: project.id, - namespace_id: project.namespace_id + user: user, + project: project, + namespace: project.namespace ) end diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb deleted file mode 100644 index dc0112c14d6..00000000000 --- a/lib/gitlab/utils.rb +++ /dev/null @@ -1,259 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Utils - extend self - DoubleEncodingError ||= Class.new(StandardError) - - def allowlisted?(absolute_path, allowlist) - path = absolute_path.downcase - - allowlist.map(&:downcase).any? do |allowed_path| - path.start_with?(allowed_path) - end - end - - def decode_path(encoded_path) - decoded = CGI.unescape(encoded_path) - if decoded != CGI.unescape(decoded) - raise DoubleEncodingError, "path #{encoded_path} is not allowed" - end - - decoded - end - - def force_utf8(str) - str.dup.force_encoding(Encoding::UTF_8) - end - - def ensure_utf8_size(str, bytes:) - raise ArgumentError, 'Empty string provided!' if str.empty? - raise ArgumentError, 'Negative string size provided!' if bytes < 0 - - truncated = str.each_char.each_with_object(+'') do |char, object| - if object.bytesize + char.bytesize > bytes - break object - else - object.concat(char) - end - end - - truncated + ('0' * (bytes - truncated.bytesize)) - end - - # Append path to host, making sure there's one single / in between - def append_path(host, path) - "#{host.to_s.sub(%r{\/+$}, '')}/#{remove_leading_slashes(path)}" - end - - def remove_leading_slashes(str) - str.to_s.sub(%r{^/+}, '') - end - - # A slugified version of the string, suitable for inclusion in URLs and - # domain names. Rules: - # - # * Lowercased - # * Anything not matching [a-z0-9-] is replaced with a - - # * Maximum length is 63 bytes - # * First/Last Character is not a hyphen - def slugify(str) - str.downcase - .gsub(/[^a-z0-9]/, '-')[0..62] - .gsub(/(\A-+|-+\z)/, '') - end - - # Converts newlines into HTML line break elements - def nlbr(str) - ActionView::Base.full_sanitizer.sanitize(+str, tags: []).gsub(/\r?\n/, '<br>').html_safe - end - - def remove_line_breaks(str) - str.gsub(/\r?\n/, '') - end - - def to_boolean(value, default: nil) - value = value.to_s if [0, 1].include?(value) - - return value if [true, false].include?(value) - return true if value =~ /^(true|t|yes|y|1|on)$/i - return false if value =~ /^(false|f|no|n|0|off)$/i - - default - end - - def boolean_to_yes_no(bool) - if bool - 'Yes' - else - 'No' - end - end - - # Behaves like `which` on Linux machines: given PATH, try to resolve the given - # executable name to an absolute path, or return nil. - # - # which('ruby') #=> /usr/bin/ruby - def which(filename) - ENV['PATH']&.split(File::PATH_SEPARATOR)&.each do |path| - full_path = File.join(path, filename) - return full_path if File.executable?(full_path) - end - - nil - end - - def try_megabytes_to_bytes(size) - Integer(size).megabytes - rescue ArgumentError - size - end - - def bytes_to_megabytes(bytes) - bytes.to_f / Numeric::MEGABYTE - end - - def ms_to_round_sec(ms) - (ms.to_f / 1000).round(6) - end - - # Used in EE - # Accepts either an Array or a String and returns an array - def ensure_array_from_string(string_or_array) - return string_or_array if string_or_array.is_a?(Array) - - string_or_array.split(',').map(&:strip) - end - - def deep_indifferent_access(data) - case data - when Array - data.map(&method(:deep_indifferent_access)) - when Hash - data.with_indifferent_access - else - data - end - end - - def deep_symbolized_access(data) - case data - when Array - data.map(&method(:deep_symbolized_access)) - when Hash - data.deep_symbolize_keys - else - data - end - end - - def string_to_ip_object(str) - return unless str - - IPAddr.new(str) - rescue IPAddr::InvalidAddressError - end - - # A safe alternative to String#downcase! - # - # This will make copies of frozen strings but downcase unfrozen - # strings in place, reducing allocations. - def safe_downcase!(str) - if str.frozen? - str.downcase - else - str.downcase! || str - end - end - - # Converts a string to an Addressable::URI object. - # If the string is not a valid URI, it returns nil. - # Param uri_string should be a String object. - # This method returns an Addressable::URI object or nil. - def parse_url(uri_string) - Addressable::URI.parse(uri_string) - rescue Addressable::URI::InvalidURIError, TypeError - end - - def add_url_parameters(url, params) - uri = parse_url(url.to_s) - uri.query_values = uri.query_values.to_h.merge(params.to_h.stringify_keys) - uri.query_values = nil if uri.query_values.empty? - uri.to_s - end - - def removes_sensitive_data_from_url(uri_string) - uri = parse_url(uri_string) - - return unless uri - return uri_string unless uri.fragment - - stripped_params = CGI.parse(uri.fragment) - if stripped_params['access_token'] - stripped_params['access_token'] = 'filtered' - filtered_query = Addressable::URI.new - filtered_query.query_values = stripped_params - - uri.fragment = filtered_query.query - end - - uri.to_s - end - - # Invert a hash, collecting all keys that map to a given value in an array. - # - # Unlike `Hash#invert`, where the last encountered pair wins, and which has the - # type `Hash[k, v] => Hash[v, k]`, `multiple_key_invert` does not lose any - # information, has the type `Hash[k, v] => Hash[v, Array[k]]`, and the original - # hash can always be reconstructed. - # - # example: - # - # multiple_key_invert({ a: 1, b: 2, c: 1 }) - # # => { 1 => [:a, :c], 2 => [:b] } - # - def multiple_key_invert(hash) - hash.flat_map { |k, v| Array.wrap(v).zip([k].cycle) } - .group_by(&:first) - .transform_values { |kvs| kvs.map(&:last) } - end - - # This sort is stable (see https://en.wikipedia.org/wiki/Sorting_algorithm#Stability) - # contrary to the bare Ruby sort_by method. Using just sort_by leads to - # instability across different platforms (e.g., x86_64-linux and x86_64-darwin18) - # which in turn leads to different sorting results for the equal elements across - # these platforms. - # This method uses a list item's original index position to break ties. - def stable_sort_by(list) - list.sort_by.with_index { |x, idx| [yield(x), idx] } - end - - # Check for valid brackets (`[` and `]`) in a string using this aspects: - # * open brackets count == closed brackets count - # * (optionally) reject nested brackets via `allow_nested: false` - # * open / close brackets coherence, eg. ][[] -> invalid - def valid_brackets?(string = '', allow_nested: true) - # remove everything except brackets - brackets = string.remove(/[^\[\]]/) - - return true if brackets.empty? - # balanced counts check - return false if brackets.size.odd? - - unless allow_nested - # nested brackets check - return false if brackets.include?('[[') || brackets.include?(']]') - end - - # open / close brackets coherence check - untrimmed = brackets - loop do - trimmed = untrimmed.gsub('[]', '') - return true if trimmed.empty? - return false if trimmed == untrimmed - - untrimmed = trimmed - end - end - end -end diff --git a/lib/gitlab/utils/override.rb b/lib/gitlab/utils/override.rb index 1d02bcbb2d2..10370811bb5 100644 --- a/lib/gitlab/utils/override.rb +++ b/lib/gitlab/utils/override.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_relative '../utils' +require 'gitlab/utils/all' require_relative '../environment' module Gitlab diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb deleted file mode 100644 index 2b3841b8f09..00000000000 --- a/lib/gitlab/utils/strong_memoize.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Utils - module StrongMemoize - # Instead of writing patterns like this: - # - # def trigger_from_token - # return @trigger if defined?(@trigger) - # - # @trigger = Ci::Trigger.find_by_token(params[:token].to_s) - # end - # - # We could write it like: - # - # include Gitlab::Utils::StrongMemoize - # - # def trigger_from_token - # Ci::Trigger.find_by_token(params[:token].to_s) - # end - # strong_memoize_attr :trigger_from_token - # - # def enabled? - # Feature.enabled?(:some_feature) - # end - # strong_memoize_attr :enabled? - # - def strong_memoize(name) - key = ivar(name) - - if instance_variable_defined?(key) - instance_variable_get(key) - else - instance_variable_set(key, yield) - end - end - - # Works the same way as "strong_memoize" but takes - # a second argument - expire_in. This allows invalidate - # the data after specified number of seconds - def strong_memoize_with_expiration(name, expire_in) - key = ivar(name) - expiration_key = "#{key}_expired_at" - - if instance_variable_defined?(expiration_key) - expire_at = instance_variable_get(expiration_key) - clear_memoization(name) if Time.current > expire_at - end - - if instance_variable_defined?(key) - instance_variable_get(key) - else - value = instance_variable_set(key, yield) - instance_variable_set(expiration_key, Time.current + expire_in) - value - end - end - - def strong_memoize_with(name, *args) - container = strong_memoize(name) { {} } - - if container.key?(args) - container[args] - else - container[args] = yield - end - end - - def strong_memoized?(name) - key = ivar(StrongMemoize.normalize_key(name)) - instance_variable_defined?(key) - end - - def clear_memoization(name) - key = ivar(StrongMemoize.normalize_key(name)) - remove_instance_variable(key) if instance_variable_defined?(key) - end - - module StrongMemoizeClassMethods - def strong_memoize_attr(method_name) - member_name = StrongMemoize.normalize_key(method_name) - - StrongMemoize.send(:do_strong_memoize, self, method_name, member_name) # rubocop:disable GitlabSecurity/PublicSend - end - end - - def self.included(base) - base.singleton_class.prepend(StrongMemoizeClassMethods) - end - - private - - # Convert `"name"`/`:name` into `:@name` - # - # Depending on a type ensure that there's a single memory allocation - def ivar(name) - case name - when Symbol - name.to_s.prepend("@").to_sym - when String - :"@#{name}" - else - raise ArgumentError, "Invalid type of '#{name}'" - end - end - - class << self - def normalize_key(key) - return key unless key.end_with?('!', '?') - - # Replace invalid chars like `!` and `?` with allowed Unicode codeparts. - key.to_s.tr('!?', "\uFF01\uFF1F") - end - - private - - def do_strong_memoize(klass, method_name, member_name) - method = klass.instance_method(method_name) - - unless method.arity == 0 - raise <<~ERROR - Using `strong_memoize_attr` on methods with parameters is not supported. - - Use `strong_memoize_with` instead. - See https://docs.gitlab.com/ee/development/utilities.html#strongmemoize - ERROR - end - - # Methods defined within a class method are already public by default, so we don't need to - # explicitly make them public. - scope = %i[private protected].find do |scope| - klass.send("#{scope}_instance_methods") # rubocop:disable GitlabSecurity/PublicSend - .include? method_name - end - - klass.define_method(method_name) do |&block| - strong_memoize(member_name) do - method.bind_call(self, &block) - end - end - - klass.send(scope, method_name) if scope # rubocop:disable GitlabSecurity/PublicSend - end - end - end - end -end diff --git a/lib/gitlab/utils/usage_data.rb b/lib/gitlab/utils/usage_data.rb index 4106084b301..1e482901929 100644 --- a/lib/gitlab/utils/usage_data.rb +++ b/lib/gitlab/utils/usage_data.rb @@ -240,7 +240,7 @@ module Gitlab yield.merge(key => Time.current) end - # @param event_name [String] the event name + # @param event_name [String, Symbol] the event name # @param values [Array|String] the values counted def track_usage_event(event_name, values) Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name.to_s, values: values) diff --git a/lib/gitlab/uuid.rb b/lib/gitlab/uuid.rb index 016c25eb94b..a3abe90a412 100644 --- a/lib/gitlab/uuid.rb +++ b/lib/gitlab/uuid.rb @@ -10,8 +10,6 @@ module Gitlab }.freeze UUID_V5_PATTERN = /\h{8}-\h{4}-5\h{3}-\h{4}-\h{12}/.freeze - NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze - PACK_PATTERN = "NnnnnN" class << self def v5(name, namespace_id: default_namespace_id) @@ -25,12 +23,7 @@ module Gitlab private def default_namespace_id - @default_namespace_id ||= begin - namespace_uuid = NAMESPACE_IDS.fetch(Rails.env.to_sym) - # Digest::UUID is broken when using a UUID as a namespace_id - # https://github.com/rails/rails/issues/37681#issue-520718028 - namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN) - end + NAMESPACE_IDS.fetch(Rails.env.to_sym) end end end diff --git a/lib/gitlab/version_info.rb b/lib/gitlab/version_info.rb deleted file mode 100644 index 0351c9b30b3..00000000000 --- a/lib/gitlab/version_info.rb +++ /dev/null @@ -1,98 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class VersionInfo - include Comparable - - attr_reader :major, :minor, :patch - - VERSION_REGEX = /(\d+)\.(\d+)\.(\d+)/.freeze - # To mitigate ReDoS, limit the length of the version string we're - # willing to check - MAX_VERSION_LENGTH = 128 - - def self.parse(str, parse_suffix: false) - if str.is_a?(self) - str - elsif str && str.length <= MAX_VERSION_LENGTH && m = str.match(VERSION_REGEX) - VersionInfo.new(m[1].to_i, m[2].to_i, m[3].to_i, parse_suffix ? m.post_match : nil) - else - VersionInfo.new - end - end - - def initialize(major = 0, minor = 0, patch = 0, suffix = nil) - @major = major - @minor = minor - @patch = patch - @suffix_s = suffix.to_s - end - - # rubocop:disable Metrics/CyclomaticComplexity - # rubocop:disable Metrics/PerceivedComplexity - def <=>(other) - return unless other.is_a? VersionInfo - return unless valid? && other.valid? - - if other.major < @major - 1 - elsif @major < other.major - -1 - elsif other.minor < @minor - 1 - elsif @minor < other.minor - -1 - elsif other.patch < @patch - 1 - elsif @patch < other.patch - -1 - elsif @suffix_s.empty? && other.suffix.present? - 1 - elsif other.suffix.empty? && @suffix_s.present? - -1 - else - suffix <=> other.suffix - end - end - # rubocop:enable Metrics/CyclomaticComplexity - # rubocop:enable Metrics/PerceivedComplexity - - def to_s - if valid? - "%d.%d.%d%s" % [@major, @minor, @patch, @suffix_s] - else - 'Unknown' - end - end - - def to_json(*_args) - { major: @major, minor: @minor, patch: @patch }.to_json - end - - def suffix - @suffix ||= @suffix_s.strip.gsub('-', '.pre.').scan(/\d+|[a-z]+/i).map do |s| - /^\d+$/ =~ s ? s.to_i : s - end.freeze - end - - def valid? - @major >= 0 && @minor >= 0 && @patch >= 0 && @major + @minor + @patch > 0 - end - - def hash - [self.class, to_s].hash - end - - def eql?(other) - (self <=> other) == 0 - end - - def same_minor_version?(other) - @major == other.major && @minor == other.minor - end - - def without_patch - self.class.new(@major, @minor, 0) - end - end -end diff --git a/lib/gitlab/web_hooks.rb b/lib/gitlab/web_hooks.rb index 8c6de56292a..031f69f3679 100644 --- a/lib/gitlab/web_hooks.rb +++ b/lib/gitlab/web_hooks.rb @@ -4,5 +4,6 @@ module Gitlab module WebHooks GITLAB_EVENT_HEADER = 'X-Gitlab-Event' GITLAB_INSTANCE_HEADER = 'X-Gitlab-Instance' + GITLAB_UUID_HEADER = 'X-Gitlab-Webhook-UUID' end end |