diff options
author | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
---|---|---|
committer | GitLab Bot <gitlab-bot@gitlab.com> | 2023-11-14 11:41:52 +0300 |
commit | 585826cb22ecea5998a2c2a4675735c94bdeedac (patch) | |
tree | 5b05f0b30d33cef48963609e8a18a4dff260eab3 /lib/gitlab | |
parent | df221d036e5d0c6c0ee4d55b9c97f481ee05dee8 (diff) |
Add latest changes from gitlab-org/gitlab@16-6-stable-eev16.6.0-rc42
Diffstat (limited to 'lib/gitlab')
184 files changed, 9940 insertions, 1100 deletions
diff --git a/lib/gitlab/analytics/cycle_analytics/request_params.rb b/lib/gitlab/analytics/cycle_analytics/request_params.rb index cea25ba2db4..0c4a0afa1d5 100644 --- a/lib/gitlab/analytics/cycle_analytics/request_params.rb +++ b/lib/gitlab/analytics/cycle_analytics/request_params.rb @@ -203,7 +203,8 @@ module Gitlab def validate_date_range return if created_after.nil? || created_before.nil? - if (created_before - created_after) > MAX_RANGE_DAYS + time_period = created_before.at_beginning_of_day - created_after.at_beginning_of_day + if time_period > MAX_RANGE_DAYS errors.add(:created_after, s_('CycleAnalytics|The given date range is larger than 180 days')) end end diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb index bf3f5b61825..469927b8a53 100644 --- a/lib/gitlab/application_rate_limiter.rb +++ b/lib/gitlab/application_rate_limiter.rb @@ -55,6 +55,7 @@ module Gitlab phone_verification_send_code: { threshold: 10, interval: 1.hour }, phone_verification_verify_code: { threshold: 10, interval: 10.minutes }, namespace_exists: { threshold: 20, interval: 1.minute }, + update_namespace_name: { threshold: -> { application_settings.update_namespace_name_rate_limit }, interval: 1.hour }, fetch_google_ip_list: { threshold: 10, interval: 1.minute }, project_fork_sync: { threshold: 10, interval: 30.minutes }, ai_action: { threshold: 160, interval: 8.hours }, diff --git a/lib/gitlab/auth.rb b/lib/gitlab/auth.rb index fc1f7a1583c..578cfb52714 100644 --- a/lib/gitlab/auth.rb +++ b/lib/gitlab/auth.rb @@ -60,10 +60,10 @@ module Gitlab Gitlab.config.omniauth.enabled end - def find_for_git_client(login, password, project:, ip:) - raise "Must provide an IP for rate limiting" if ip.nil? + def find_for_git_client(login, password, project:, request:) + raise "Must provide an IP for rate limiting" if request.ip.nil? - rate_limiter = Gitlab::Auth::IpRateLimiter.new(ip) + rate_limiter = Gitlab::Auth::IpRateLimiter.new(request.ip) raise IpBlocked if !skip_rate_limit?(login: login) && rate_limiter.banned? @@ -80,7 +80,7 @@ module Gitlab user_with_password_for_git(login, password) || Gitlab::Auth::Result::EMPTY - rate_limit!(rate_limiter, success: result.success?, login: login) + rate_limit!(rate_limiter, success: result.success?, login: login, request: request) look_to_limit_user(result.actor) return result if result.success? || authenticate_using_internal_or_ldap_password? @@ -142,7 +142,7 @@ module Gitlab private - def rate_limit!(rate_limiter, success:, login:) + def rate_limit!(rate_limiter, success:, login:, request:) return if skip_rate_limit?(login: login) if success @@ -155,8 +155,18 @@ module Gitlab # request from this IP if needed. # This returns true when the failures are over the threshold and the IP # is banned. - Gitlab::AppLogger.info "IP #{rate_limiter.ip} failed to login " \ - "as #{login} but has been temporarily banned from Git auth" + + message = "Rack_Attack: Git auth failures has exceeded the threshold. " \ + "IP has been temporarily banned from Git auth." + + Gitlab::AuthLogger.error( + message: message, + env: :blocklist, + remote_ip: request.ip, + request_method: request.request_method, + path: request.fullpath, + login: login + ) end end diff --git a/lib/gitlab/auth/saml/config.rb b/lib/gitlab/auth/saml/config.rb index 7524d8b9f85..e6c9f04eff5 100644 --- a/lib/gitlab/auth/saml/config.rb +++ b/lib/gitlab/auth/saml/config.rb @@ -8,6 +8,21 @@ module Gitlab def enabled? ::AuthHelper.saml_providers.any? end + + def default_attribute_statements + defaults = OmniAuth::Strategies::SAML.default_options[:attribute_statements].to_hash.deep_symbolize_keys + defaults[:nickname] = %w[username nickname] + defaults[:name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/name' + defaults[:name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/name' + defaults[:email] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/emailaddress' + defaults[:email] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/emailaddress' + defaults[:first_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/givenname' + defaults[:first_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/givenname' + defaults[:last_name] << 'http://schemas.xmlsoap.org/ws/2005/05/identity/claims/surname' + defaults[:last_name] << 'http://schemas.microsoft.com/ws/2008/06/identity/claims/surname' + + defaults + end end DEFAULT_PROVIDER_NAME = 'saml' diff --git a/lib/gitlab/auth/two_factor_auth_verifier.rb b/lib/gitlab/auth/two_factor_auth_verifier.rb index fbdfd105ee3..4b66aaf0e6a 100644 --- a/lib/gitlab/auth/two_factor_auth_verifier.rb +++ b/lib/gitlab/auth/two_factor_auth_verifier.rb @@ -36,7 +36,7 @@ module Gitlab return false unless time - two_factor_grace_period.hours.since(time) < Time.current + two_factor_grace_period.hours.since(time).past? end def allow_2fa_bypass_for_provider diff --git a/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb new file mode 100644 index 00000000000..04fd09f81f0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_packages_tags_project_id.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # This migration populates the new `packages_tags.project_id` column from joining with `packages_packages` table + class BackfillPackagesTagsProjectId < BatchedMigrationJob + operation_name :update_all # This is used as the key on collecting metrics + scope_to ->(relation) { relation.where(project_id: nil) } + feature_category :package_registry + + def perform + each_sub_batch do |sub_batch| + joined = sub_batch + .joins('INNER JOIN packages_packages ON packages_tags.package_id = packages_packages.id') + .select('packages_tags.id, packages_packages.project_id') + + ApplicationRecord.connection.execute <<~SQL + WITH joined_cte(packages_tag_id, project_id) AS MATERIALIZED ( + #{joined.to_sql} + ) + UPDATE packages_tags + SET project_id = joined_cte.project_id + FROM joined_cte + WHERE id = joined_cte.packages_tag_id + SQL + end + end + end + end +end diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 952e6d01f1a..9e9fc9b98b7 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -130,7 +130,7 @@ module Gitlab end def base_relation - define_batchable_model(batch_table, connection: connection) + define_batchable_model(batch_table, connection: connection, primary_key: batch_column) .where(batch_column => start_id..end_id) end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb new file mode 100644 index 00000000000..99bc638532a --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_merge_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_branch_merge_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedBranchMergeAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_branch_merge_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_branch_merge_access_levels.group_id + AND pgl.project_id = protected_branches.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb new file mode 100644 index 00000000000..a6934cf5adc --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_branch_push_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_branch_push_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedBranchPushAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_branch_push_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_branches ON protected_branches.id = protected_branch_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_branch_push_access_levels.group_id + AND pgl.project_id = protected_branches.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb new file mode 100644 index 00000000000..8c59e42a9f6 --- /dev/null +++ b/lib/gitlab/background_migration/delete_invalid_protected_tag_create_access_levels.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to remove protected_tag_create_access_levels for groups that do not have project_group_links + # to the project for the associated protected branch + class DeleteInvalidProtectedTagCreateAccessLevels < BatchedMigrationJob + operation_name :delete_invalid_protected_tag_create_access_levels + scope_to ->(relation) { relation.where.not(group_id: nil) } + feature_category :source_code_management + + def perform + each_sub_batch do |sub_batch| + sub_batch + .joins('INNER JOIN protected_tags ON protected_tags.id = protected_tag_id') + .joins(%( + LEFT OUTER JOIN project_group_links pgl + ON pgl.group_id = protected_tag_create_access_levels.group_id + AND pgl.project_id = protected_tags.project_id + )) + .where(%( + pgl.id IS NULL + )).delete_all + end + end + end + end +end diff --git a/lib/gitlab/base_doorkeeper_controller.rb b/lib/gitlab/base_doorkeeper_controller.rb index 91994c2fa95..c8520993b8e 100644 --- a/lib/gitlab/base_doorkeeper_controller.rb +++ b/lib/gitlab/base_doorkeeper_controller.rb @@ -3,7 +3,8 @@ # This is a base controller for doorkeeper. # It adds the `can?` helper used in the views. module Gitlab - class BaseDoorkeeperController < BaseActionController + # rubocop:disable Rails/ApplicationController + class BaseDoorkeeperController < ActionController::Base include Gitlab::Allowable include EnforcesTwoFactorAuthentication include SessionsHelper @@ -12,4 +13,5 @@ module Gitlab helper_method :can? end + # rubocop:enable Rails/ApplicationController end diff --git a/lib/gitlab/bitbucket_import/importers/issue_importer.rb b/lib/gitlab/bitbucket_import/importers/issue_importer.rb index 2c3be67eabc..d194a311278 100644 --- a/lib/gitlab/bitbucket_import/importers/issue_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issue_importer.rb @@ -40,6 +40,8 @@ module Gitlab project.issues.create!(attributes) + metrics.issues_counter.increment + log_info(import_stage: 'import_issue', message: 'finished', iid: object[:iid]) rescue StandardError => e track_import_failure!(project, exception: e) diff --git a/lib/gitlab/bitbucket_import/importers/issues_importer.rb b/lib/gitlab/bitbucket_import/importers/issues_importer.rb index 6162433e701..8ab82ddb0be 100644 --- a/lib/gitlab/bitbucket_import/importers/issues_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/issues_importer.rb @@ -7,17 +7,21 @@ module Gitlab include ParallelScheduling def execute + return job_waiter unless repo.issues_enabled? + log_info(import_stage: 'import_issues', message: 'importing issues') issues = client.issues(project.import_source) labels = build_labels_hash - issues.each do |issue| + issues.each_with_index do |issue, index| job_waiter.jobs_remaining += 1 next if already_enqueued?(issue) + allocate_issues_internal_id! if index == 0 + job_delay = calculate_job_delay(job_waiter.jobs_remaining) issue_hash = issue.to_hash.merge({ issue_type_id: default_issue_type_id, label_id: labels[issue.kind] }) @@ -49,11 +53,23 @@ module Gitlab ::WorkItems::Type.default_issue_type.id end + def allocate_issues_internal_id! + last_bitbucket_issue = client.last_issue(repo) + + return unless last_bitbucket_issue + + Issue.track_namespace_iid!(project.project_namespace, last_bitbucket_issue.iid) + end + def build_labels_hash labels = {} project.labels.each { |l| labels[l.title.to_s] = l.id } labels end + + def repo + @repo ||= client.repo(project.import_source) + end end end end diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb index a18d50e8fce..f7b1753a9f9 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_importer.rb @@ -45,6 +45,8 @@ module Gitlab merge_request.assignee_ids = [author_id] merge_request.reviewer_ids = reviewers merge_request.save! + + metrics.merge_requests_counter.increment end log_info(import_stage: 'import_pull_request', message: 'finished', iid: object[:iid]) diff --git a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb index 8ea8b1562f2..934e4ee1720 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_request_notes_importer.rb @@ -4,21 +4,22 @@ module Gitlab module BitbucketImport module Importers class PullRequestNotesImporter - include Loggable - include ErrorTracking + include ParallelScheduling def initialize(project, hash) @project = project - @importer = Gitlab::BitbucketImport::Importer.new(project) + @formatter = Gitlab::ImportFormatter.new + @user_finder = UserFinder.new(project) + @ref_converter = Gitlab::BitbucketImport::RefConverter.new(project) @object = hash.with_indifferent_access + @position_map = {} + @discussion_map = {} end def execute log_info(import_stage: 'import_pull_request_notes', message: 'starting', iid: object[:iid]) - merge_request = project.merge_requests.find_by(iid: object[:iid]) # rubocop: disable CodeReuse/ActiveRecord - - importer.import_pull_request_comments(merge_request, merge_request) if merge_request + import_pull_request_comments if merge_request log_info(import_stage: 'import_pull_request_notes', message: 'finished', iid: object[:iid]) rescue StandardError => e @@ -27,7 +28,116 @@ module Gitlab private - attr_reader :object, :project, :importer + attr_reader :object, :project, :formatter, :user_finder, :ref_converter, :discussion_map, :position_map + + def import_pull_request_comments + inline_comments, pr_comments = comments.partition(&:inline?) + + import_inline_comments(inline_comments) + import_standalone_pr_comments(pr_comments) + end + + def import_inline_comments(inline_comments) + children, parents = inline_comments.partition(&:has_parent?) + + parents.each do |comment| + position_map[comment.iid] = build_position(comment) + + import_comment(comment) + end + + children.each do |comment| + position_map[comment.iid] = position_map.fetch(comment.parent_id, nil) + + import_comment(comment) + end + end + + def import_comment(comment) + position = position_map[comment.iid] + discussion_id = discussion_map[comment.parent_id] + + note = create_diff_note(comment, position, discussion_id) + + discussion_map[comment.iid] = note&.discussion_id + end + + def create_diff_note(comment, position, discussion_id) + attributes = pull_request_comment_attributes(comment) + attributes.merge!(position: position, type: 'DiffNote', discussion_id: discussion_id) + + note = merge_request.notes.build(attributes) + + return note if note.save + + # Bitbucket supports the ability to comment on any line, not just the + # line in the diff. If we can't add the note as a DiffNote, fallback to creating + # a regular note. + + log_info(import_stage: 'create_diff_note', message: 'creating fallback DiffNote', iid: merge_request.iid) + create_fallback_diff_note(comment, position) + rescue StandardError => e + Gitlab::ErrorTracking.log_exception( + e, + import_stage: 'create_diff_note', comment_id: comment.iid, error: e.message + ) + + nil + end + + def create_fallback_diff_note(comment, position) + attributes = pull_request_comment_attributes(comment) + note = "*Comment on" + + note += " #{position.old_path}:#{position.old_line} -->" if position&.old_line + note += " #{position.new_path}:#{position.new_line}" if position&.new_line + note += "*\n\n#{comment.note}" + + attributes[:note] = note + merge_request.notes.create!(attributes) + end + + def build_position(pr_comment) + params = { + diff_refs: merge_request.diff_refs, + old_path: pr_comment.file_path, + new_path: pr_comment.file_path, + old_line: pr_comment.old_pos, + new_line: pr_comment.new_pos + } + + Gitlab::Diff::Position.new(params) + end + + def import_standalone_pr_comments(pr_comments) + pr_comments.each do |comment| + attributes = pull_request_comment_attributes(comment) + merge_request.notes.create!(attributes) + end + end + + def pull_request_comment_attributes(comment) + { + project: project, + author_id: user_finder.gitlab_user_id(project, comment.author), + note: comment_note(comment), + created_at: comment.created_at, + updated_at: comment.updated_at + } + end + + def comment_note(comment) + author = formatter.author_line(comment.author) unless user_finder.find_user_id(comment.author) + author.to_s + ref_converter.convert_note(comment.note.to_s) + end + + def merge_request + @merge_request ||= project.merge_requests.iid_in(object[:iid]).first + end + + def comments + client.pull_request_comments(project.import_source, merge_request.iid).reject(&:deleted?) + end end end end diff --git a/lib/gitlab/bitbucket_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_import/importers/repository_importer.rb index b8c0ba69d37..9be7ed99436 100644 --- a/lib/gitlab/bitbucket_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/repository_importer.rb @@ -19,6 +19,7 @@ module Gitlab validate_repository_size! + set_default_branch update_clone_time end @@ -76,6 +77,16 @@ module Gitlab def validate_repository_size! # Defined in EE end + + def set_default_branch + default_branch = client.repo(project.import_source).default_branch + + project.change_head(default_branch) if default_branch + end + + def client + Bitbucket::Client.new(project.import_data.credentials) + end end end end diff --git a/lib/gitlab/bitbucket_import/loggable.rb b/lib/gitlab/bitbucket_import/loggable.rb index eda3cc96d4d..aeae993b9eb 100644 --- a/lib/gitlab/bitbucket_import/loggable.rb +++ b/lib/gitlab/bitbucket_import/loggable.rb @@ -19,6 +19,10 @@ module Gitlab logger.error(log_data(messages)) end + def metrics + Gitlab::Import::Metrics.new(:bitbucket_importer, project) + end + private def logger diff --git a/lib/gitlab/checks/diff_check.rb b/lib/gitlab/checks/diff_check.rb index 15b38188f13..a359236e150 100644 --- a/lib/gitlab/checks/diff_check.rb +++ b/lib/gitlab/checks/diff_check.rb @@ -31,8 +31,7 @@ module Gitlab def treeish_objects objects = commits - return objects unless project.repository.empty? && - Feature.enabled?(:verify_push_rules_for_first_commit, project) + return objects unless project.repository.empty? # It's a special case for the push to the empty repository # diff --git a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb index 35f969dbb46..b8c6bdee1bb 100644 --- a/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb +++ b/lib/gitlab/checks/file_size_check/any_oversized_blobs.rb @@ -6,7 +6,7 @@ module Gitlab class AnyOversizedBlobs def initialize(project:, changes:, file_size_limit_megabytes:) @project = project - @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord just plucking from an array + @newrevs = changes.pluck(:newrev).compact # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck @file_size_limit_megabytes = file_size_limit_megabytes end diff --git a/lib/gitlab/ci/ansi2json/line.rb b/lib/gitlab/ci/ansi2json/line.rb index 21fc2980cdc..791b8a963e9 100644 --- a/lib/gitlab/ci/ansi2json/line.rb +++ b/lib/gitlab/ci/ansi2json/line.rb @@ -35,13 +35,15 @@ module Gitlab end attr_reader :offset, :sections, :segments, :current_segment, - :section_header, :section_duration, :section_options + :section_header, :section_footer, :section_duration, + :section_options def initialize(offset:, style:, sections: []) @offset = offset @segments = [] @sections = sections @section_header = false + @section_footer = false @duration = nil @current_segment = Segment.new(style: style) end @@ -79,6 +81,10 @@ module Gitlab @section_header = true end + def set_as_section_footer + @section_footer = true + end + def set_section_duration(duration_in_seconds) normalized_duration_in_seconds = duration_in_seconds.to_i.clamp(0, 1.year) duration = ActiveSupport::Duration.build(normalized_duration_in_seconds) @@ -103,6 +109,7 @@ module Gitlab { offset: offset, content: @segments }.tap do |result| result[:section] = sections.last if sections.any? result[:section_header] = true if @section_header + result[:section_footer] = true if @section_footer result[:section_duration] = @section_duration if @section_duration result[:section_options] = @section_options if @section_options end diff --git a/lib/gitlab/ci/ansi2json/state.rb b/lib/gitlab/ci/ansi2json/state.rb index 3aec1cde1bc..6cf76fbbb51 100644 --- a/lib/gitlab/ci/ansi2json/state.rb +++ b/lib/gitlab/ci/ansi2json/state.rb @@ -49,6 +49,7 @@ module Gitlab duration = timestamp.to_i - @open_sections[section].to_i @current_line.set_section_duration(duration) + @current_line.set_as_section_footer @open_sections.delete(section) end diff --git a/lib/gitlab/ci/build/context/build.rb b/lib/gitlab/ci/build/context/build.rb index 48b138b0258..bbcdcd7d389 100644 --- a/lib/gitlab/ci/build/context/build.rb +++ b/lib/gitlab/ci/build/context/build.rb @@ -33,13 +33,9 @@ module Gitlab # Assigning tags and needs is slow and they are not needed for rules # evaluation since we don't use them to compute the variables at this point. def build_attributes - if pipeline.reduced_build_attributes_list_for_rules? - attributes - .except(:tag_list, :needs_attributes) - .merge!(pipeline_attributes, ci_stage_attributes) - else - attributes.merge(pipeline_attributes, ci_stage_attributes) - end + attributes + .except(:tag_list, :needs_attributes) + .merge!(pipeline_attributes, ci_stage_attributes) end def ci_stage_attributes diff --git a/lib/gitlab/ci/components/instance_path.rb b/lib/gitlab/ci/components/instance_path.rb index df2b2a14fc6..50731d54fc0 100644 --- a/lib/gitlab/ci/components/instance_path.rb +++ b/lib/gitlab/ci/components/instance_path.rb @@ -18,7 +18,6 @@ module Gitlab def initialize(address:) @full_path, @version = address.to_s.split('@', 2) @host = Settings.gitlab_ci['component_fqdn'] - @component_project = ::Ci::Catalog::ComponentsProject.new(project, sha) end def fetch_content!(current_user:) @@ -27,7 +26,8 @@ module Gitlab raise Gitlab::Access::AccessDeniedError unless Ability.allowed?(current_user, :download_code, project) - @component_project.fetch_component(component_name) + component_project = ::Ci::Catalog::ComponentsProject.new(project, sha) + component_project.fetch_component(component_name) end def project diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index bf8a99ef45e..5fcafcba829 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ module Gitlab ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks coverage retry parallel interruptible timeout - release id_tokens publish].freeze + release id_tokens publish pages].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -40,13 +40,19 @@ module Gitlab if needs_value[:job].nil? && needs_value[:cross_dependency].present? errors.add(:needs, "corresponding to dependencies must be from the same pipeline") else - missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord (Array#pluck) + missing_needs = dependencies - needs_value[:job].pluck(:name) # rubocop:disable CodeReuse/ActiveRecord -- Array#pluck errors.add(:dependencies, "the #{missing_needs.join(", ")} should be part of needs") if missing_needs.any? end end - validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? } + validates :publish, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } + + validates :pages, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } end entry :before_script, Entry::Commands, @@ -127,10 +133,14 @@ module Gitlab description: 'Path to be published with Pages', inherit: false + entry :pages, ::Gitlab::Ci::Config::Entry::Pages, + inherit: false, + description: 'Pages configuration.' + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, - :interruptible, :timeout, - :release, :allow_failure, :publish + :interruptible, :timeout, :release, + :allow_failure, :publish, :pages def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -170,7 +180,8 @@ module Gitlab needs: needs_defined? ? needs_value : nil, scheduling_type: needs_defined? ? :dag : :stage, id_tokens: id_tokens_value, - publish: publish + publish: publish, + pages: pages ).compact end diff --git a/lib/gitlab/ci/config/entry/pages.rb b/lib/gitlab/ci/config/entry/pages.rb new file mode 100644 index 00000000000..57d9e944f51 --- /dev/null +++ b/lib/gitlab/ci/config/entry/pages.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the pages path prefix + # Entry that represents the pages attributes + # + class Pages < ::Gitlab::Config::Entry::Node + ALLOWED_KEYS = %i[path_prefix].freeze + + include ::Gitlab::Config::Entry::Attributable + include ::Gitlab::Config::Entry::Validatable + + attributes ALLOWED_KEYS + + validations do + validates :config, type: Hash + validates :config, allowed_keys: ALLOWED_KEYS + + with_options allow_nil: true do + validates :path_prefix, type: String + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 88734ac1186..d0e9a9afc51 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -25,6 +25,8 @@ module Gitlab validates :name, type: Symbol validates :name, length: { maximum: 255 } + validates :config, mutually_exclusive_keys: %i[script trigger] + validates :config, disallowed_keys: { in: %i[only except start_in], message: 'key may not be used with `rules`', diff --git a/lib/gitlab/ci/config/header/input.rb b/lib/gitlab/ci/config/header/input.rb index dcb96006459..08ee70b6290 100644 --- a/lib/gitlab/ci/config/header/input.rb +++ b/lib/gitlab/ci/config/header/input.rb @@ -11,17 +11,24 @@ module Gitlab include ::Gitlab::Config::Entry::Validatable include ::Gitlab::Config::Entry::Attributable - ALLOWED_KEYS = %i[default description regex type].freeze + ALLOWED_KEYS = %i[default description options regex type].freeze + ALLOWED_OPTIONS_LIMIT = 50 attributes ALLOWED_KEYS, prefix: :input validations do validates :config, type: Hash, allowed_keys: ALLOWED_KEYS validates :key, alphanumeric: true - validates :input_default, alphanumeric: true, allow_nil: true validates :input_description, alphanumeric: true, allow_nil: true validates :input_regex, type: String, allow_nil: true validates :input_type, allow_nil: true, allowed_values: Interpolation::Inputs.input_types + validates :input_options, type: Array, allow_nil: true + + validate do + if input_options&.size.to_i > ALLOWED_OPTIONS_LIMIT + errors.add(:config, "cannot define more than #{ALLOWED_OPTIONS_LIMIT} options") + end + end end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb index ba519776635..987268b0525 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/base_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/base_input.rb @@ -20,11 +20,6 @@ module Gitlab raise NotImplementedError end - # Checks whether the provided value is of the given type - def valid_value?(value) - raise NotImplementedError - end - attr_reader :errors, :name, :spec, :value def initialize(name:, spec:, value:) @@ -54,20 +49,39 @@ module Gitlab private def validate! - return error('required value has not been provided') if required_input? && value.nil? + validate_required + + return if errors.present? - # validate default value - if !required_input? && !valid_value?(default) - return error("default value is not a #{self.class.type_name}") - end + run_validations(default, default: true) unless required_input? - # validate provided value - return error("provided value is not a #{self.class.type_name}") unless valid_value?(actual_value) + run_validations(value) unless value.nil? + end + + def validate_required + error('required value has not been provided') if required_input? && value.nil? + end - validate_regex! + def run_validations(value, default: false) + validate_type(value, default) + validate_options(value) + validate_regex(value, default) end - def validate_regex! + # Type validations are done separately for different input types. + def validate_type(_value, _default) + raise NotImplementedError + end + + # Options can be either StringInput or NumberInput and are validated accordingly. + def validate_options(_value) + return unless options + + error('Options can only be used with string and number inputs') + end + + # Regex can be only be a StringInput and is validated accordingly. + def validate_regex(_value, _default) return unless spec.key?(:regex) error('RegEx validation can only be used with string inputs') @@ -96,6 +110,10 @@ module Gitlab def default spec[:default] end + + def options + spec[:options] + end end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb index 0293c01a5a8..4c34f7e7fdd 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/boolean_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class BooleanInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) spec.is_a?(Hash) && spec[:type] == type_name end @@ -14,8 +16,11 @@ module Gitlab 'boolean' end - def valid_value?(value) - [true, false].include?(value) + override :validate_type + def validate_type(value, default) + return if [true, false].include?(value) + + error("#{default ? 'default' : 'provided'} value is not a boolean") end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb index 314315d2b6d..59bc057749a 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/number_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/number_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class NumberInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) spec.is_a?(Hash) && spec[:type] == type_name end @@ -14,8 +16,19 @@ module Gitlab 'number' end - def valid_value?(value) - value.is_a?(Numeric) + override :validate_type + def validate_type(value, default) + return if value.is_a?(Numeric) + + error("#{default ? 'default' : 'provided'} value is not a number") + end + + override :validate_options + def validate_options(value) + return unless options && value + return if options.include?(value) + + error("`#{value}` cannot be used because it is not in the list of the allowed options") end end end diff --git a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb index 3f40e851f11..01b9d34a883 100644 --- a/lib/gitlab/ci/config/interpolation/inputs/string_input.rb +++ b/lib/gitlab/ci/config/interpolation/inputs/string_input.rb @@ -6,6 +6,8 @@ module Gitlab module Interpolation class Inputs class StringInput < BaseInput + extend ::Gitlab::Utils::Override + def self.matches?(spec) # The input spec can be `nil` when using a minimal specification # and also when `type` is not specified. @@ -22,24 +24,32 @@ module Gitlab 'string' end - def valid_value?(value) - value.nil? || value.is_a?(String) + override :validate_type + def validate_type(value, default) + return if value.is_a?(String) + + error("#{default ? 'default' : 'provided'} value is not a string") + end + + override :validate_options + def validate_options(value) + return unless options && value + return if options.include?(value) + + error("`#{value}` cannot be used because it is not in the list of allowed options") end private - def validate_regex! + override :validate_regex + def validate_regex(value, default) return unless spec.key?(:regex) safe_regex = ::Gitlab::UntrustedRegexp.new(spec[:regex]) - return if safe_regex.match?(actual_value) + return if safe_regex.match?(value) - if value.nil? - error('default value does not match required RegEx pattern') - else - error('provided value does not match required RegEx pattern') - end + error("#{default ? 'default' : 'provided'} value does not match required RegEx pattern") rescue RegexpError error('invalid regular expression') end diff --git a/lib/gitlab/ci/jwt_v2.rb b/lib/gitlab/ci/jwt_v2.rb index 29beba4774a..90db9d13d85 100644 --- a/lib/gitlab/ci/jwt_v2.rb +++ b/lib/gitlab/ci/jwt_v2.rb @@ -25,13 +25,26 @@ module Gitlab def reserved_claims super.merge({ - iss: Settings.gitlab.base_url, + iss: Feature.enabled?(:oidc_issuer_url) ? Gitlab.config.gitlab.url : Settings.gitlab.base_url, sub: "project_path:#{project.full_path}:ref_type:#{ref_type}:ref:#{source_ref}", - aud: aud, - user_identities: user_identities + aud: aud }.compact) end + def custom_claims + additional_custom_claims = { + runner_id: runner&.id, + runner_environment: runner_environment, + sha: pipeline.sha, + project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level), + user_identities: user_identities + }.compact + + mapper = ClaimMapper.new(project_config, pipeline) + + super.merge(additional_custom_claims).merge(mapper.to_h) + end + def user_identities return unless user&.pass_user_identities_to_ci_jwt @@ -43,17 +56,6 @@ module Gitlab end end - def custom_claims - mapper = ClaimMapper.new(project_config, pipeline) - - super.merge({ - runner_id: runner&.id, - runner_environment: runner_environment, - sha: pipeline.sha, - project_visibility: Gitlab::VisibilityLevel.string_level(project.visibility_level) - }).merge(mapper.to_h) - end - def project_config Gitlab::Ci::ProjectConfig.new( project: project, diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb index 3dc73544208..35548358c57 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx_properties.rb @@ -15,7 +15,8 @@ module Gitlab SUPPORTED_SCHEMA_VERSION = '1' GITLAB_PREFIX = 'gitlab:' SOURCE_PARSERS = { - 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning + 'dependency_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::DependencyScanning, + 'container_scanning' => ::Gitlab::Ci::Parsers::Sbom::Source::ContainerScanning }.freeze SUPPORTED_PROPERTIES = %w[ meta:schema_version @@ -24,6 +25,10 @@ module Gitlab dependency_scanning:source_file:path dependency_scanning:package_manager:name dependency_scanning:language:name + container_scanning:image:name + container_scanning:image:tag + container_scanning:operating_system:name + container_scanning:operating_system:version ].freeze def self.parse_source(...) diff --git a/lib/gitlab/ci/parsers/sbom/source/base_source.rb b/lib/gitlab/ci/parsers/sbom/source/base_source.rb new file mode 100644 index 00000000000..744555aa25a --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/base_source.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class BaseSource + REQUIRED_ATTRIBUTES = [].freeze + + def self.source(...) + new(...).source + end + + def initialize(data) + @data = data + end + + def source + return unless required_attributes_present? + + ::Gitlab::Ci::Reports::Sbom::Source.new( + type: type, + data: data + ) + end + + private + + attr_reader :data + + # Implement in child class + # returns a symbol of the source type + def type; end + + def required_attributes_present? + self.class::REQUIRED_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb new file mode 100644 index 00000000000..33f9631c424 --- /dev/null +++ b/lib/gitlab/ci/parsers/sbom/source/container_scanning.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Sbom + module Source + class ContainerScanning < BaseSource + REQUIRED_ATTRIBUTES = [ + %w[image name], + %w[image tag] + ].freeze + + OPERATING_SYSTEM_ATTRIBUTES = [ + %w[operating_system name], + %w[operating_system version] + ].freeze + + private + + def type + :container_scanning + end + + def required_attributes_present? + operating_system_attributes_valid? && super + end + + def operating_system_attributes_valid? + return true if data['operating_system'].blank? + + OPERATING_SYSTEM_ATTRIBUTES.all? do |keys| + data.dig(*keys).present? + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb index c76a4309779..fc5a7606e39 100644 --- a/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb +++ b/lib/gitlab/ci/parsers/sbom/source/dependency_scanning.rb @@ -5,36 +5,15 @@ module Gitlab module Parsers module Sbom module Source - class DependencyScanning + class DependencyScanning < BaseSource REQUIRED_ATTRIBUTES = [ %w[input_file path] ].freeze - def self.source(...) - new(...).source - end - - def initialize(data) - @data = data - end - - def source - return unless required_attributes_present? - - ::Gitlab::Ci::Reports::Sbom::Source.new( - type: :dependency_scanning, - data: data - ) - end - private - attr_reader :data - - def required_attributes_present? - REQUIRED_ATTRIBUTES.all? do |keys| - data.dig(*keys).present? - end + def type + :dependency_scanning end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 9032faa66d4..be6c6c2558b 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -141,7 +141,7 @@ module Gitlab project_id: @project.id, found_by_pipeline: report.pipeline, vulnerability_finding_signatures_enabled: @signatures_enabled, - cvss: data['cvss'] || [] + cvss: data['cvss_vectors'] || [] ) ) end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index e39482481c7..e2a8044b708 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6], - secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6] + cluster_image_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + container_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + coverage_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + dast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + api_fuzzing: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + dependency_scanning: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + sast: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7], + secret_detection: %w[15.0.0 15.0.1 15.0.2 15.0.4 15.0.5 15.0.6 15.0.7] }.freeze VERSIONS_TO_REMOVE_IN_17_0 = %w[].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..e27096d071f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/cluster-image-scanning-report-format.json @@ -0,0 +1,1085 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json new file mode 100644 index 00000000000..94c3b3fc919 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/container-scanning-report-format.json @@ -0,0 +1,1017 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..e15fbc3ed56 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/coverage-fuzzing-report-format.json @@ -0,0 +1,975 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json new file mode 100644 index 00000000000..8a9519f442f --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dast-report-format.json @@ -0,0 +1,1380 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json new file mode 100644 index 00000000000..83b3537b5f1 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/dependency-scanning-report-format.json @@ -0,0 +1,1083 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json new file mode 100644 index 00000000000..3597ed169d5 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/sast-report-format.json @@ -0,0 +1,970 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json new file mode 100644 index 00000000000..afd80ca916b --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.7/secret-detection-report-format.json @@ -0,0 +1,994 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n![GitLab Logo](https://about.gitlab.com/images/press/logo/preview/gitlab-logo-white-preview.png)" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.7" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "options": { + "type": "array", + "items": { + "type": "object", + "description": "A configuration option used for this scan.", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "description": "The configuration option name.", + "maxLength": 255, + "minLength": 1, + "examples": [ + "DAST_FF_ENABLE_BAS", + "DOCKER_TLS_CERTDIR", + "DS_MAX_DEPTH", + "SECURE_LOG_LEVEL" + ] + }, + "source": { + "type": "string", + "description": "The source of this option.", + "enum": [ + "argument", + "file", + "env_variable", + "other" + ] + }, + "value": { + "type": [ + "boolean", + "integer", + "null", + "string" + ], + "description": "The value used for this scan.", + "examples": [ + true, + 2, + null, + "fatal", + "" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^(https?|ftp)://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "cvss_vectors": { + "type": "array", + "minItems": 1, + "maxItems": 10, + "description": "An ordered array of CVSS vectors, each issued by a vendor to rate the vulnerability. The first item in the array is used as the primary CVSS vector, and is used to filter and sort the vulnerability.", + "items": { + "oneOf": [ + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 16, + "maxLength": 128, + "pattern": "^((AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))/)*(AV:[NAL]|AC:[LMH]|Au:[MSN]|[CIA]:[NPC]|E:(U|POC|F|H|ND)|RL:(OF|TF|W|U|ND)|RC:(UC|UR|C|ND)|CDP:(N|L|LM|MH|H|ND)|TD:(N|L|M|H|ND)|[CIA]R:(L|M|H|ND))$" + } + }, + "required": [ + "vendor", + "vector" + ] + }, + { + "type": "object", + "properties": { + "vendor": { + "type": "string", + "minLength": 1, + "default": "unknown" + }, + "vector": { + "type": "string", + "minLength": 32, + "maxLength": 128, + "pattern": "^CVSS:3[.][01]/((AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])/)*(AV:[NALP]|AC:[LH]|PR:[NLH]|UI:[NR]|S:[UC]|[CIA]:[NLH]|E:[XUPFH]|RL:[XOTWU]|RC:[XURC]|[CIA]R:[XLMH]|MAV:[XNALP]|MAC:[XLH]|MPR:[XNLH]|MUI:[XNR]|MS:[XUC]|M[CIA]:[XNLH])$" + } + }, + "required": [ + "vendor", + "vector" + ] + } + ] + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^(https?|ftp)://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/status/build/cancelable.rb b/lib/gitlab/ci/status/build/cancelable.rb index 43fb5cdbbe6..b8c8cfa802c 100644 --- a/lib/gitlab/ci/status/build/cancelable.rb +++ b/lib/gitlab/ci/status/build/cancelable.rb @@ -6,7 +6,7 @@ module Gitlab module Build class Cancelable < Status::Extended def has_action? - can?(user, :update_build, subject) + can?(user, :cancel_build, subject) end def action_icon diff --git a/lib/gitlab/ci/status/composite.rb b/lib/gitlab/ci/status/composite.rb index 1ba78b357e5..fe4f6db9549 100644 --- a/lib/gitlab/ci/status/composite.rb +++ b/lib/gitlab/ci/status/composite.rb @@ -61,6 +61,8 @@ module Gitlab 'running' elsif any_of?(:waiting_for_resource) 'waiting_for_resource' + elsif any_of?(:waiting_for_callback) + 'waiting_for_callback' elsif any_of?(:manual) 'manual' elsif any_of?(:scheduled) diff --git a/lib/gitlab/ci/status/waiting_for_callback.rb b/lib/gitlab/ci/status/waiting_for_callback.rb new file mode 100644 index 00000000000..0184a910ede --- /dev/null +++ b/lib/gitlab/ci/status/waiting_for_callback.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Status + class WaitingForCallback < Status::Core + def text + s_('CiStatusText|Waiting') + end + + def label + s_('CiStatusLabel|waiting for callback') + end + + def icon + 'status_pending' + end + + def favicon + 'favicon_status_pending' + end + + def group + 'waiting-for-callback' + end + + def details_path + nil + end + end + end + end +end diff --git a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml index 356062c734e..324128678de 100644 --- a/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Cosign.gitlab-ci.yml @@ -12,9 +12,9 @@ include: docker-build: variables: - COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations + COSIGN_YES: "true" # Used by Cosign to skip confirmation prompts for non-destructive operations id_tokens: - SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio + SIGSTORE_ID_TOKEN: # Used by Cosign to get certificate from Fulcio aud: sigstore after_script: - apk add --update cosign diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 2d04c97b32e..6898923bc53 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 2d04c97b32e..6898923bc53 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.44.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.49.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index 4d53b92763a..7d923245d79 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 390824e8e49..0f8d5bf6d8f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index a9681c0f927..e29d18ea45a 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.59.1' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.60.0' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml index d7a6104082d..4c89497fa97 100644 --- a/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Kaniko.gitlab-ci.yml @@ -46,13 +46,30 @@ kaniko-build: # Write credentials to access Gitlab Container Registry within the runner/ci - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64 | tr -d '\n')\"}}}" > /kaniko/.docker/config.json # Build and push the container. To disable push add --no-push - - DOCKERFILE_PATH=${DOCKERFILE_PATH:-"$KANIKO_BUILD_CONTEXT/Dockerfile"} + # Both Dockerfile and Containerfile are supported. For retrocompatibility, if both files are present, Dockerfile will be used. + - | + if [ -z "$DOCKERFILE_PATH" ]; then + if [ -f "$KANIKO_BUILD_CONTEXT/Dockerfile" ]; then + DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Dockerfile" + elif [ -n "$CONTAINERFILE_PATH" ]; then + DOCKERFILE_PATH="$CONTAINERFILE_PATH" + elif [ -f "$KANIKO_BUILD_CONTEXT/Containerfile" ]; then + DOCKERFILE_PATH="$KANIKO_BUILD_CONTEXT/Containerfile" + else \ + echo "No suitable configuration for the build context have been found. Please check your configuration." + exit 1 + fi + fi + - echo $DOCKERFILE_PATH - /kaniko/executor --context $KANIKO_BUILD_CONTEXT --dockerfile $DOCKERFILE_PATH --destination $IMAGE_TAG $KANIKO_ARGS - # Run this job in a branch/tag where a Dockerfile exists + # Run this job in a branch/tag where a Containerfile/Dockerfile exists rules: - exists: + - Containerfile - Dockerfile - # custom Dockerfile path + # custom Containerfile/Dockerfile path + # If both variables are set, DOCKERFILE_PATH will be used - if: $DOCKERFILE_PATH + - if: $CONTAINERFILE_PATH # custom build context without an explicit Dockerfile path - if: $KANIKO_BUILD_CONTEXT != $CI_PROJECT_DIR diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index d2b929cf995..0ba4f9715c5 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -50,7 +50,11 @@ variables: - gitlab-terraform plan-json resource_group: ${TF_STATE_NAME} artifacts: - # The next line, which disables public access to pipeline artifacts, may not be available everywhere. + # Terraform's cache files can include secrets which can be accidentally exposed. + # Please exercise caution when utilizing secrets in your Terraform infrastructure and + # consider limiting access to artifacts or take other security measures to protect sensitive information. + # + # The next line, which disables public access to pipeline artifacts, is not available on GitLab.com. # See: https://docs.gitlab.com/ee/ci/yaml/#artifactspublic public: false paths: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index c1a90955f7f..8c9e0a329dd 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -19,6 +19,7 @@ browser_performance: SITESPEED_IMAGE: sitespeedio/sitespeed.io SITESPEED_VERSION: 26.1.0 SITESPEED_OPTIONS: '' + SITESPEED_DOCKER_OPTIONS: '' services: - docker:dind script: @@ -48,7 +49,7 @@ browser_performance: HTTP_PROXY \ NO_PROXY \ ) \ - --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index adc92fde5ae..3f4c0c53850 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -19,6 +19,7 @@ browser_performance: SITESPEED_IMAGE: sitespeedio/sitespeed.io SITESPEED_VERSION: latest SITESPEED_OPTIONS: '' + SITESPEED_DOCKER_OPTIONS: '' services: - docker:dind script: @@ -48,7 +49,7 @@ browser_performance: HTTP_PROXY \ NO_PROXY \ ) \ - --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS + $SITESPEED_DOCKER_OPTIONS --shm-size=1g --rm -v "$(pwd)":/sitespeed.io $SITESPEED_IMAGE:$SITESPEED_VERSION --plugins.add ./gitlab-exporter --cpu --outputFolder sitespeed-results $URL $SITESPEED_OPTIONS - mv sitespeed-results/data/performance.json browser-performance.json artifacts: paths: diff --git a/lib/gitlab/ci/yaml_processor/dag.rb b/lib/gitlab/ci/yaml_processor/dag.rb index 4a122c73e80..d3047385c99 100644 --- a/lib/gitlab/ci/yaml_processor/dag.rb +++ b/lib/gitlab/ci/yaml_processor/dag.rb @@ -17,13 +17,15 @@ module Gitlab def self.check_circular_dependencies!(jobs) new(jobs).tsort - rescue TSort::Cyclic - raise ValidationError, 'The pipeline has circular dependencies' + rescue TSort::Cyclic => e + raise ValidationError, "The pipeline has circular dependencies: #{e.message}" end def tsort_each_child(node, &block) return unless @nodes[node] + raise TSort::Cyclic, "self-dependency: #{node}" if @nodes[node].include?(node) + @nodes[node].each(&block) end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6207b595fc6..2435d128bf2 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -8,8 +8,7 @@ module Gitlab class Result attr_reader :errors, :warnings, :root_variables, :root_variables_with_prefill_data, - :stages, :jobs, - :workflow_rules, :workflow_name + :stages, :jobs, :workflow_rules, :workflow_name def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @@ -124,7 +123,8 @@ module Gitlab trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, release: job[:release], - publish: job[:publish] + publish: job[:publish], + pages: job[:pages] }.compact }.compact end diff --git a/lib/gitlab/composer/version_index.rb b/lib/gitlab/composer/version_index.rb index bfa3112b795..0834fda9cf9 100644 --- a/lib/gitlab/composer/version_index.rb +++ b/lib/gitlab/composer/version_index.rb @@ -48,8 +48,7 @@ module Gitlab end def package_source(package) - use_http_url = package.project.public? || Feature.disabled?(:composer_use_ssh_source_urls, package.project) - git_url = use_http_url ? package.project.http_url_to_repo : package.project.ssh_url_to_repo + git_url = package.project.public? ? package.project.http_url_to_repo : package.project.ssh_url_to_repo { 'type' => 'git', diff --git a/lib/gitlab/config/entry/validators.rb b/lib/gitlab/config/entry/validators.rb index 87b7cab3f6d..c7dd11b0432 100644 --- a/lib/gitlab/config/entry/validators.rb +++ b/lib/gitlab/config/entry/validators.rb @@ -56,8 +56,7 @@ module Gitlab mutually_exclusive_keys = value.try(:keys).to_a & options[:in] if mutually_exclusive_keys.length > 1 - record.errors.add(attribute, "please use only one of the following keys: " + - mutually_exclusive_keys.join(', ')) + record.errors.add(attribute, "these keys cannot be used together: #{mutually_exclusive_keys.join(', ')}") end end end diff --git a/lib/gitlab/config_checker/puma_rugged_checker.rb b/lib/gitlab/config_checker/puma_rugged_checker.rb deleted file mode 100644 index 82c59f3328b..00000000000 --- a/lib/gitlab/config_checker/puma_rugged_checker.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module ConfigChecker - module PumaRuggedChecker - extend self - extend Gitlab::Git::RuggedImpl::UseRugged - - def check - notices = [] - - if running_puma_with_multiple_threads? && rugged_enabled_through_feature_flag? - link_start = '<a href="https://docs.gitlab.com/ee/administration/operations/puma.html#performance-caveat-when-using-puma-with-rugged">' - link_end = '</a>' - notices << { - type: 'warning', - message: _('Puma is running with a thread count above 1 and the Rugged '\ - 'service is enabled. This may decrease performance in some environments. '\ - 'See our %{link_start}documentation%{link_end} '\ - 'for details of this issue.') % { link_start: link_start, link_end: link_end } - } - end - - notices - end - end - end -end diff --git a/lib/gitlab/database/dictionary.rb b/lib/gitlab/database/dictionary.rb new file mode 100644 index 00000000000..7b0c8560a26 --- /dev/null +++ b/lib/gitlab/database/dictionary.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module Gitlab + module Database + class Dictionary + def initialize(file_path) + @file_path = file_path + @data = YAML.load_file(file_path) + end + + def name_and_schema + [key_name, gitlab_schema.to_sym] + end + + def table_name + data['table_name'] + end + + def view_name + data['view_name'] + end + + def milestone + data['milestone'] + end + + def gitlab_schema + data['gitlab_schema'] + end + + def schema?(schema_name) + gitlab_schema == schema_name.to_s + end + + def key_name + table_name || view_name + end + + def validate! + return true unless gitlab_schema.nil? + + raise( + GitlabSchema::UnknownSchemaError, + "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ + "See #{help_page_url}" + ) + end + + private + + attr_reader :file_path, :data + + def help_page_url + # rubocop:disable Gitlab/DocUrl -- link directly to docs.gitlab.com, always + 'https://docs.gitlab.com/ee/development/database/database_dictionary.html' + # rubocop:enable Gitlab/DocUrl + end + end + end +end diff --git a/lib/gitlab/database/dynamic_model_helpers.rb b/lib/gitlab/database/dynamic_model_helpers.rb index 83edf77f37e..18854530278 100644 --- a/lib/gitlab/database/dynamic_model_helpers.rb +++ b/lib/gitlab/database/dynamic_model_helpers.rb @@ -5,7 +5,7 @@ module Gitlab module DynamicModelHelpers BATCH_SIZE = 1_000 - def define_batchable_model(table_name, connection:) + def define_batchable_model(table_name, connection:, primary_key: nil) klass = Class.new(ActiveRecord::Base) do include EachBatch @@ -13,6 +13,7 @@ module Gitlab self.inheritance_column = :_type_disabled end + klass.primary_key = primary_key if connection.primary_keys(table_name).length > 1 klass.connection = connection klass end diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 31ceb898eee..ecb45622061 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -31,6 +31,7 @@ module Gitlab '_test_gitlab_main_cell_' => :gitlab_main_cell, '_test_gitlab_main_' => :gitlab_main, '_test_gitlab_ci_' => :gitlab_ci, + '_test_gitlab_jh_' => :gitlab_jh, '_test_gitlab_embedding_' => :gitlab_embedding, '_test_gitlab_geo_' => :gitlab_geo, '_test_gitlab_pm_' => :gitlab_pm, @@ -138,19 +139,19 @@ module Gitlab end def self.deleted_tables_to_schema - @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').to_h + @deleted_tables_to_schema ||= self.build_dictionary('deleted_tables').map(&:name_and_schema).to_h end def self.deleted_views_to_schema - @deleted_views_to_schema ||= self.build_dictionary('deleted_views').to_h + @deleted_views_to_schema ||= self.build_dictionary('deleted_views').map(&:name_and_schema).to_h end def self.tables_to_schema - @tables_to_schema ||= self.build_dictionary('').to_h + @tables_to_schema ||= self.build_dictionary('').map(&:name_and_schema).to_h end def self.views_to_schema - @views_to_schema ||= self.build_dictionary('views').to_h + @views_to_schema ||= self.build_dictionary('views').map(&:name_and_schema).to_h end def self.schema_names @@ -159,21 +160,9 @@ module Gitlab def self.build_dictionary(scope) Dir.glob(dictionary_path_globs(scope)).map do |file_path| - data = YAML.load_file(file_path) - - key_name = data['table_name'] || data['view_name'] - - # rubocop:disable Gitlab/DocUrl - if data['gitlab_schema'].nil? - raise( - UnknownSchemaError, - "#{file_path} must specify a valid gitlab_schema for #{key_name}. " \ - "See https://docs.gitlab.com/ee/development/database/database_dictionary.html" - ) - end - # rubocop:enable Gitlab/DocUrl - - [key_name, data['gitlab_schema'].to_sym] + dictionary = Dictionary.new(file_path) + dictionary.validate! + dictionary end end end diff --git a/lib/gitlab/database/migration.rb b/lib/gitlab/database/migration.rb index 41044816de9..1d17c2ca608 100644 --- a/lib/gitlab/database/migration.rb +++ b/lib/gitlab/database/migration.rb @@ -56,6 +56,10 @@ module Gitlab include Gitlab::Database::Migrations::RunnerBackoff::MigrationHelpers end + class V2_2 < V2_1 + include Gitlab::Database::Migrations::MilestoneMixin + end + def self.[](version) version = version.to_s name = "V#{version.tr('.', '_')}" @@ -66,7 +70,7 @@ module Gitlab # The current version to be used in new migrations def self.current_version - 2.1 + 2.2 end end end diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index efcceafda90..a57bce789c7 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -18,8 +18,8 @@ module Gitlab include AsyncConstraints::MigrationHelpers include WraparoundVacuumHelpers - def define_batchable_model(table_name, connection: self.connection) - super(table_name, connection: connection) + def define_batchable_model(table_name, connection: self.connection, primary_key: nil) + super(table_name, connection: connection, primary_key: primary_key) end def each_batch(table_name, connection: self.connection, **kwargs) @@ -821,6 +821,7 @@ module Gitlab primary_key: :id, batch_size: 20_000, sub_batch_size: 1000, + pause_ms: 100, interval: 2.minutes ) @@ -848,6 +849,7 @@ module Gitlab conversions.keys, conversions.values, job_interval: interval, + pause_ms: pause_ms, batch_size: batch_size, sub_batch_size: sub_batch_size) end diff --git a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb index 11f1e62e8b9..d1edb739b85 100644 --- a/lib/gitlab/database/migration_helpers/convert_to_bigint.rb +++ b/lib/gitlab/database/migration_helpers/convert_to_bigint.rb @@ -4,12 +4,16 @@ module Gitlab module Database module MigrationHelpers module ConvertToBigint - # This helper is extracted for the purpose of - # https://gitlab.com/gitlab-org/gitlab/-/issues/392815 - # so that we can test all combinations just once, - # and simplify migration tests. - # - # Once we are done with the PK conversions we can remove this. + INDEX_OPTIONS_MAP = { + unique: :unique, + order: :orders, + opclass: :opclasses, + where: :where, + type: :type, + using: :using, + comment: :comment + }.freeze + def com_or_dev_or_test_but_not_jh? return true if Gitlab.dev_or_test_env? @@ -29,6 +33,78 @@ module Gitlab column.sql_type == 'bigint' && temp_column.sql_type == 'integer' end + + def add_bigint_column_indexes(table_name, int_column_name) + bigint_column_name = convert_to_bigint_column(int_column_name) + + unless column_exists?(table_name.to_s, bigint_column_name) + raise "Bigint column '#{bigint_column_name}' does not exist on #{table_name}" + end + + indexes(table_name).each do |i| + next unless Array(i.columns).join(' ').match?(/\b#{int_column_name}\b/) + + create_bigint_index(table_name, i, int_column_name, bigint_column_name) + end + end + + # default 'index_name' method is not used because this method can be reused while swapping/dropping the indexes + def bigint_index_name(int_column_index_name) + # First 20 digits of the hash is chosen to make sure it fits the 63 chars limit + digest = Digest::SHA256.hexdigest(int_column_index_name).first(20) + "bigint_idx_#{digest}" + end + + private + + def create_bigint_index(table_name, index_definition, int_column_name, bigint_column_name) + index_attributes = index_definition.as_json + index_options = INDEX_OPTIONS_MAP + .transform_values { |key| index_attributes[key.to_s] } + .select { |_, v| v.present? } + + bigint_index_options = create_bigint_options( + index_options, + index_definition.name, + int_column_name, + bigint_column_name + ) + + add_concurrent_index( + table_name, + bigint_index_columns(int_column_name, bigint_column_name, index_definition.columns), + name: bigint_index_options.delete(:name), + ** bigint_index_options + ) + end + + def bigint_index_columns(int_column_name, bigint_column_name, int_index_columns) + if int_index_columns.is_a?(String) + int_index_columns.gsub(/\b#{int_column_name}\b/, bigint_column_name) + else + int_index_columns.map do |column| + column == int_column_name.to_s ? bigint_column_name : column + end + end + end + + def create_bigint_options(index_options, int_index_name, int_column_name, bigint_column_name) + index_options[:name] = bigint_index_name(int_index_name) + index_options[:where]&.gsub!(/\b#{int_column_name}\b/, bigint_column_name) + + # ordering on multiple columns will return a Hash instead of string + index_options[:order] = + if index_options[:order].is_a?(Hash) + index_options[:order].to_h do |column, order| + column = bigint_column_name if column == int_column_name + [column, order] + end + else + index_options[:order]&.gsub(/\b#{int_column_name}\b/, bigint_column_name) + end + + index_options.select { |_, v| v.present? } + end end end end diff --git a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb index 555efb58606..7f215bc0db7 100644 --- a/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb +++ b/lib/gitlab/database/migration_helpers/wraparound_autovacuum.rb @@ -13,7 +13,7 @@ module Gitlab # 3. Introduce the migration again for self-managed. # def can_execute_on?(*tables) - return false unless Gitlab.com? || Gitlab.dev_or_test_env? + return false unless Gitlab.com_except_jh? || Gitlab.dev_or_test_env? if wraparound_prevention_on_tables?(tables) Gitlab::AppLogger.info(message: "Wraparound prevention vacuum detected", class: self.class) diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 64cde273a59..3d4ac113bf6 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -72,6 +72,7 @@ module Gitlab batch_max_value: nil, batch_class_name: BATCH_CLASS_NAME, batch_size: BATCH_SIZE, + pause_ms: 100, max_batch_size: nil, sub_batch_size: SUB_BATCH_SIZE, gitlab_schema: nil @@ -105,6 +106,7 @@ module Gitlab column_name: batch_column_name, job_arguments: job_arguments, interval: job_interval, + pause_ms: pause_ms, min_value: batch_min_value, max_value: batch_max_value, batch_class_name: batch_class_name, diff --git a/lib/gitlab/database/migrations/milestone_mixin.rb b/lib/gitlab/database/migrations/milestone_mixin.rb index 10bc0c192e7..7d78f74d237 100644 --- a/lib/gitlab/database/migrations/milestone_mixin.rb +++ b/lib/gitlab/database/migrations/milestone_mixin.rb @@ -19,11 +19,10 @@ module Gitlab end end - def initialize(name = class_name, version = nil, type = nil) - raise MilestoneNotSetError, "Milestone is not set for #{self.class.name}" if milestone.nil? + def initialize(name = self.class.name, version = nil, _type = nil) + raise MilestoneNotSetError, "Milestone is not set for #{name}" if milestone.nil? super(name, version) - @version = Gitlab::Database::Migrations::Version.new(version, milestone, type) end def milestone # rubocop:disable Lint/DuplicateMethods diff --git a/lib/gitlab/database/partitioning/partition_manager.rb b/lib/gitlab/database/partitioning/partition_manager.rb index bb70d052e3e..83cd446534c 100644 --- a/lib/gitlab/database/partitioning/partition_manager.rb +++ b/lib/gitlab/database/partitioning/partition_manager.rb @@ -89,6 +89,8 @@ module Gitlab Gitlab::AppLogger.info(message: "Created partition", partition_name: partition.partition_name, table_name: partition.table) + + lock_partitions_for_writes(partition) if should_lock_for_writes? end model.partitioning_strategy.after_adding_partitions @@ -205,6 +207,23 @@ module Gitlab end end end + + def should_lock_for_writes? + Feature.enabled?(:automatic_lock_writes_on_partition_tables, type: :ops) && + Gitlab::Database.database_mode == Gitlab::Database::MODE_MULTIPLE_DATABASES && + connection != model.connection + end + strong_memoize_attr :should_lock_for_writes? + + def lock_partitions_for_writes(partition) + table_name = "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{partition.partition_name}" + Gitlab::Database::LockWritesManager.new( + table_name: table_name, + connection: connection, + database_name: @connection_name, + with_retries: !connection.transaction_open? + ).lock_writes + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 1ce0a44e37f..b486ddb8e76 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -8,7 +8,8 @@ module Gitlab include ::Gitlab::Database::MigrationHelpers include ::Gitlab::Database::MigrationHelpers::LooseForeignKeyHelpers - ALLOWED_TABLES = %w[audit_events web_hook_logs].freeze + ALLOWED_TABLES = %w[audit_events web_hook_logs merge_request_diff_files merge_request_diff_commits].freeze + ERROR_SCOPE = 'table partitioning' MIGRATION_CLASS_NAME = "::#{module_parent_name}::BackfillPartitionedTable" @@ -16,6 +17,60 @@ module Gitlab BATCH_INTERVAL = 2.minutes.freeze BATCH_SIZE = 50_000 SUB_BATCH_SIZE = 2_500 + PARTITION_BUFFER = 6 + MIN_ID = 1 + + # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a int/bigint column. + # One partition is created per partition_size between 1 and MAX(column_name). Also installs a trigger on + # the original table to copy writes into the partitioned table. To copy over historic data from before creation + # of the partitioned table, use the `enqueue_partitioning_data_migration` helper in a post-deploy migration. + # Note: If the original table is empty the system creates 6 partitions in the new table. + # + # A copy of the original table is required as PG currently does not support partitioning existing tables. + # + # Example: + # + # partition_table_by_int_range :merge_request_diff_commits, :merge_request_diff_id, partition_size: 500, primary_key: ['merge_request_diff_id', 'relative_order'] + # + # Options are: + # :partition_size - a int specifying the partition size + # :primary_key - a array specifying the primary query of the new table + # + # Note: The system always adds a buffer of 6 partitions. + def partition_table_by_int_range(table_name, column_name, partition_size:, primary_key:) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_ddl_mode! + + assert_table_is_allowed(table_name) + + assert_not_in_transaction_block(scope: ERROR_SCOPE) + + current_primary_key = Array.wrap(connection.primary_key(table_name)) + raise "primary key not defined for #{table_name}" if current_primary_key.blank? + + partition_column = find_column_definition(table_name, column_name) + raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil? + + primary_key = Array.wrap(primary_key).map(&:to_s) + raise "the partition column must be part of the primary key" unless primary_key.include?(column_name.to_s) + + primary_key_objects = connection.columns(table_name).select { |column| primary_key.include?(column.name) } + + raise 'partition_size must be greater than 1' unless partition_size > 1 + + max_id = Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.with_suppressed do + Gitlab::Database::QueryAnalyzers::GitlabSchemasValidateConnection.with_suppressed do + define_batchable_model(table_name, connection: connection).maximum(column_name) || partition_size * PARTITION_BUFFER + end + end + + partitioned_table_name = make_partitioned_table_name(table_name) + + with_lock_retries do + create_range_id_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key_objects) + create_int_range_partitions(partitioned_table_name, partition_size, MIN_ID, max_id) + create_trigger_to_sync_tables(table_name, partitioned_table_name, current_primary_key) + end + end # Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column. # One partition is created per month between the given `min_date` and `max_date`. Also installs a trigger on @@ -332,6 +387,34 @@ module Gitlab connection.columns(table).find { |c| c.name == column.to_s } end + def create_range_id_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_keys) + if table_exists?(partitioned_table_name) + Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \ + " (this may be due to an aborted migration or similar): table_name: #{partitioned_table_name} " + return + end + + tmp_partitioning_column_name = "#{partition_column.name}_tmp" + + temporary_columns = primary_keys.map { |key| "#{key.name}_tmp" }.join(", ") + temporary_columns_statement = build_temporary_columns_statement(primary_keys) + + transaction do + execute(<<~SQL) + CREATE TABLE #{partitioned_table_name} ( + LIKE #{source_table_name} INCLUDING ALL EXCLUDING INDEXES, + #{temporary_columns_statement}, + PRIMARY KEY (#{temporary_columns}) + ) PARTITION BY RANGE (#{tmp_partitioning_column_name}) + SQL + + primary_keys.each do |key| + remove_column(partitioned_table_name, key.name) + rename_column(partitioned_table_name, "#{key.name}_tmp", key.name) + end + end + end + def create_range_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_key) if table_exists?(partitioned_table_name) Gitlab::AppLogger.warn "Partitioned table not created because it already exists" \ @@ -382,6 +465,20 @@ module Gitlab end end + def create_int_range_partitions(table_name, partition_size, min_id, max_id) + lower_bound = min_id + upper_bound = min_id + partition_size + + end_id = max_id + PARTITION_BUFFER * partition_size # Adds a buffer of 6 partitions + + while lower_bound < end_id + create_range_partition_safely("#{table_name}_#{lower_bound}", table_name, lower_bound, upper_bound) + + lower_bound += partition_size + upper_bound += partition_size + end + end + def to_sql_date_literal(date) connection.quote(date.strftime('%Y-%m-%d')) end @@ -411,19 +508,23 @@ module Gitlab return end + unique_key = Array.wrap(unique_key) + delimiter = ",\n " column_names = connection.columns(partitioned_table_name).map(&:name) set_statements = build_set_statements(column_names, unique_key) insert_values = column_names.map { |name| "NEW.#{name}" } + delete_where_statement = unique_key.map { |unique_key| "#{unique_key} = OLD.#{unique_key}" }.join(' AND ') + update_where_statement = unique_key.map { |unique_key| "#{partitioned_table_name}.#{unique_key} = NEW.#{unique_key}" }.join(' AND ') create_trigger_function(name, replace: false) do <<~SQL IF (TG_OP = 'DELETE') THEN - DELETE FROM #{partitioned_table_name} where #{unique_key} = OLD.#{unique_key}; + DELETE FROM #{partitioned_table_name} where #{delete_where_statement}; ELSIF (TG_OP = 'UPDATE') THEN UPDATE #{partitioned_table_name} SET #{set_statements.join(delimiter)} - WHERE #{partitioned_table_name}.#{unique_key} = NEW.#{unique_key}; + WHERE #{update_where_statement}; ELSIF (TG_OP = 'INSERT') THEN INSERT INTO #{partitioned_table_name} (#{column_names.join(delimiter)}) VALUES (#{insert_values.join(delimiter)}); @@ -433,8 +534,16 @@ module Gitlab end end + def build_temporary_columns_statement(columns) + columns.map do |column| + type = column.name == 'id' || column.name.end_with?('_id') ? 'bigint' : column.sql_type + + "#{column.name}_tmp #{type} NOT NULL" + end.join(", ") + end + def build_set_statements(column_names, unique_key) - column_names.reject { |name| name == unique_key }.map { |name| "#{name} = NEW.#{name}" } + column_names.reject { |name| unique_key.include?(name) }.map { |name| "#{name} = NEW.#{name}" } end def create_sync_trigger(table_name, trigger_name, function_name) diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb index eb55ebc7619..c2f94b7b0e6 100644 --- a/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb @@ -8,7 +8,7 @@ module Gitlab class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base RoutingTableNotUsedError = Class.new(QueryAnalyzerError) - ENABLED_TABLES = %w[ci_builds_metadata].freeze + ENABLED_TABLES = %w[ci_builds ci_builds_metadata].freeze class << self def enabled? diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb new file mode 100644 index 00000000000..583aceba098 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch < Base + SetOperatorStarError = Class.new(QueryAnalyzerError) + + DETECT_REGEX = /.*SELECT.+(UNION|EXCEPT|INTERSECT)/i + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + Feature.enabled?(:query_analyzer_gitlab_schema_metrics, type: :ops) + end + + def analyze(parsed) + return unless requires_detection?(parsed.sql) + + # Only handle SELECT queries. + parsed.pg.tree.stmts.each do |stmt| + select_stmt = next_select_stmt(stmt) + next unless select_stmt + + types = SelectStmt.new(select_stmt).types + + raise SetOperatorStarError if types.any?(Type::INVALID) + end + end + + private + + def next_select_stmt(node) + return unless node.stmt.respond_to?(:select_stmt) + + node.stmt.select_stmt + end + + # This not entirely correct and will run true on `SELECT union_station, ...` + def requires_detection?(sql) + sql.match DETECT_REGEX + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb new file mode 100644 index 00000000000..87120b8ffce --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/columns.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # Columns refer to table columns produced by queries and parts of queries. + # If we have `SELECT namespaces.id` then `id` is a column. But, we can also have + # `WHERE namespaces.id > 10` and `id` is also a column. + # + # In static analysis of a SQL query a column source can be ambiguous. + # Such as in `SELECT id FROM users, namespaces. In such cases we assume `id` could come from either `users` or + # `namespaces`. + class Columns + class << self + # Determine the type of each column in the select statement. + # Returns a Set object containing a Types enum. + # When an error is found parsing will return immediately. + def types(select_stmt) + # Forward through any errors when the column refers to a part of the SQL query that is known to include + # errors. For example, the column may refer to a column from a CTE that was invalid. + return Set.new([Type::INVALID]) if References.errors?(select_stmt.all_references) + + types = Set.new + + # Resolve the type of reference for each target in the select statement. + target_list = select_stmt.node.target_list + targets = target_list.map(&:res_target) + targets.each do |target| + target_type = get_target_type(target, select_stmt) + + # A NULL target is of the form: + # SELECT NULL::namespaces FROM namespaces + types += if Targets.null?(target) + # Maintain any errors but otherwise ignore this target. + target_type & [Type::INVALID] + else + target_type + end + end + + types + end + + private + + def get_target_type(target, select_stmt) + target_ref_names = Targets.reference_names(target, select_stmt) + + resolved_refs = References.resolved(select_stmt.all_references) + + # Cross reference column references with resolved references. + # A resolved reference is part of a SQL query that we were able to analyze already. + # A CTE or sub-query would be such a case. The only non-resolvable reference is a table. + all_resolved = (target_ref_names - resolved_refs.keys).empty? + + # Is this target `*` such as `SELECT *`. + a_star = Targets.a_star?(target) + + if all_resolved + # Defer to the reference source types. + col_refs = resolved_refs.slice(*target_ref_names) + .values + .reduce(:union) || Set.new + + if a_star + # When * the target forwards through the types of the references. + col_refs + else + # When not * the column is static, but we also forward through any nested errors. + (col_refs.to_a & [Type::INVALID]) << Type::STATIC + end + elsif a_star + # This is a * on a table. The * lookup occurs dynamically during query runtime and will + # change when the table schema changes. + [Type::DYNAMIC] + else + # This references a column on a table or intermediate result set such as: + # SELECT namespaces.id FROM namespaces + # + # or: + # WITH some_cte AS ( ... ) SELECT some_cte.id FROM some_cte + [Type::STATIC] + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb new file mode 100644 index 00000000000..0ab58ff7c6f --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/common_table_expressions.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +# The CTE in a SELECT can reference CTEs defined by the current scope, but also CTEs defined by earlier scopes. +# With the following query as an example: +# +# WITH some_cte AS (select 1) +# SELECT * +# FROM (SELECT * FROM some_cte) subquery +# +# The CTE some_cte is visible from within the subquery scope. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class CommonTableExpressions + class << self + # Convert CTEs available within this SELECT statement into a set of References. + # + # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs. + # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement. + def references(node, cte_refs) + return cte_refs if node&.with_clause.nil? + + refs = cte_refs.dup + + node.with_clause.ctes.each do |cte| + cte_name = name(cte) + cte_select_stmt = select_stmt(cte) + + # Resolve the CTE type to dynamic/static/error. + refs[cte_name] = if node.with_clause.recursive + # Recursive CTEs need special handling to avoid infinite loops. + recursive_refs(cte_refs, cte_name, cte_select_stmt) + else + SelectStmt.new(cte_select_stmt, cte_refs).types + end + end + + refs + end + + private + + def name(cte) + cte.common_table_expr.ctename + end + + def select_stmt(cte) + cte.common_table_expr.ctequery.select_stmt + end + + # Return whether the recursive CTE is dynamic/static/error. + def recursive_refs(cte_refs, cte_name, select_stmt) + # Resolve the non-recursive term before the recursive term. + larg_select_stmt = SelectStmt.new(select_stmt.larg, cte_refs) + larg_type = larg_select_stmt.types + new_cte_refs = cte_refs.merge({ cte_name => larg_type }) + + # Now we can resolve the recursive side. + rarg_type = SelectStmt.new(select_stmt.rarg, new_cte_refs).types + + final_type = larg_type | rarg_type + if final_type.count > 1 + final_type | [Type::INVALID] + else + final_type + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb new file mode 100644 index 00000000000..c205243694a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/froms.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class Froms + class << self + # Parse the FROM part of the SELECT. Construct a mapping of FROM names to their PgQuery node. Recurse any + # sub-queries and resolve to a Set of dynamic/static/error. + # + # Whenever a node is aliased, use the alias name as it's reference and ignore it's original name. + # + # For example, given: + # + # SELECT id + # FROM namespaces ns + # + # Return a Hash of { 'ns' => NodeObject } + # + # @param [PgQuery::Node] node The PgQuery SELECT statement node containing the CTEs. + # @param [References] cte_refs Inherited CTEs from scopes that wrap this SELECT statement. + # + # @return [Hash] name of from references mapped to the node that defines their value, or Set if already + # resolved. + def references(node, cte_refs) + refs = {} + + return refs unless node + + node.from_clause.each do |from| + range_var = Node.dig(from, :range_var) + range_sq = Node.dig(from, :range_subselect) + + if range_var + # FROM some_table + # FROM some_table some_alias + refs.merge!(range_var_reference(range_var, cte_refs)) + elsif Node.dig(from, :join_expr) + # FROM some_table INNER JOIN other_table + range_vars = Node.locate_descendants(from, :range_var) + range_vars.each do |range_var| + refs.merge!(range_var_reference(range_var, cte_refs)) + end + elsif range_sq + # FROM (SELECT ...) some_alias + select_stmt = Node.dig(range_sq, :subquery, :select_stmt) + refs[range_sq.alias.aliasname] = SelectStmt.new(select_stmt, cte_refs).types + end + end + + refs + end + + private + + def range_var_reference(range_var, cte_refs) + relname = Node.dig(range_var, :alias, :aliasname) || range_var.relname + reference = cte_refs[range_var.relname] || range_var + + { relname => reference } + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb new file mode 100644 index 00000000000..ee41eaa9d3a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/node.rb @@ -0,0 +1,118 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # The Node class allows us to traverse PgQuery nodes with tree like semantics. + # + # This class balances convenience and performance. The PgQuery nodes are Google::Protobuf::MessageExts which + # contain a dynamic set of attributes known as fields. Accessing these fields can cause performance problems + # due to the large volume of iterable fields. + # + # When possible use #dig over the *descendant* methods. + # + # The filter available to each method reduces the traversed attributes. The default filter only traverses nodes + # required to parse for set operator mismatches. + class Node + class << self + include Gitlab::Utils::StrongMemoize + + # The default nodes help speed up traversal. Traversal of other nodes can greatly affect performance. + DEFAULT_NODES = %i[ + a_star + alias + args + column_ref + fields + func_call + join_expr + larg + range_subselect + range_var + rarg + res_target + subquery + val + ].freeze + DEFAULT_FIELD_FILTER = ->(field) { field.is_a?(Integer) || DEFAULT_NODES.include?(field) }.freeze + + # Recurse through children. + # The block will yield the child node and the name of that node. + # Calling without a block will return an Enumerator. + def descendants(node, filter: DEFAULT_FIELD_FILTER, &blk) + if blk + children(node, filter: filter) do |child_node, child_field| + yield(child_node, child_field) + + descendants(child_node, filter: filter, &blk) + end + nil + else + enum_for(:descendants, node, filter: filter, &blk) + end + end + + # Return the first node that matches the field. + def locate_descendant(node, field, filter: DEFAULT_FIELD_FILTER) + descendants(node, filter: filter).find { |_, child_field| child_field == field }&.first + end + + # Return all nodes that match the field. + def locate_descendants(node, field, filter: DEFAULT_FIELD_FILTER) + descendants(node, filter: filter).select { |_, child_field| child_field == field }.map(&:first) + end + + # Like Hash#dig, traverse attributes in sequential order and return the final value. + # Return nil if any of the fields are not available. + def dig(node, *attrs) + obj = node + attrs.each do |attr| + if obj.respond_to?(attr) + obj = obj.public_send(attr) # rubocop:disable GitlabSecurity/PublicSend + else + obj = nil + break + end + end + obj + end + + private + + # Interface with a PgQuery result as though it was a tree node. + # All elements in a PgQuery result are ancestors of Google::Protobuf::AbstractMessage + # + # Based off PgQuery's treewalker https://github.com/pganalyze/pg_query/blob/main/lib/pg_query/treewalker.rb + def children(node, filter: DEFAULT_FIELD_FILTER, &_blk) + attributes = case node + when Google::Protobuf::MessageExts + descriptor_fields(node.class.descriptor) + when Google::Protobuf::RepeatedField + node.count.times.to_a + end + + attributes.select(&filter).each do |attr| + attr_key = attr.is_a?(Symbol) ? attr.to_s : attr + child = node[attr_key] + next if child.nil? + + yield(child, attr) + end + end + + def descriptor_fields(descriptor) + strong_memoize_with(:descriptor_fields, descriptor) do + keys = [] + descriptor.each do |field| + keys << field.name.to_sym + end + keys + end + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb new file mode 100644 index 00000000000..ba6e9752905 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/references.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +# References form the base data structure of the PreventSetOperatorMismatch query analyzer. +# +# A reference refers to a table, CTE, or other named entity in a SQL query. References are a set of mappings between the +# name of the reference and the PgQuery node that represents that reference in the parsed tree. +# +# Given the SQL: +# +# WITH some_cte AS (SELECT 1) +# SELECT * +# FROM some_cte, users, namespace ns +# +# The reference names would be `some_cte`, `users`, `ns`. The reference values are the nodes in the parse tree that +# represent that reference: +# - some_cte: the common table expression node +# - users: nil, being a table +# - ns: nil, being a table, but importantly we use the alias name +# +# A reference can be "resolved". A resolved reference value is a Set of Types. The reference value was a select +# statement that has since been parsed. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class References + class << self + # All references that have already been parsed to determine static/dynamic/error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def resolved(refs) + refs.select { |_name, ref| ref.is_a?(Set) } + end + + # All references that have not been parsed to determine static/dynamic/error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def unresolved(refs) + refs.select { |_name, ref| unresolved?(ref) } + end + + # Whether any currently resolved references have resulted in an error state. + # @param [Hash] refs A Hash of reference names mapped to the parse tree node or resolved Set of Types. + def errors?(refs) + resolved(refs).any? { |_, values| values.include?(Type::INVALID) } + end + + private + + def resolved?(ref) + ref.is_a?(Set) + end + + def unresolved?(ref) + !resolved?(ref) && table?(ref) + end + + def table?(ref) + !ref.is_a?(PgQuery::RangeVar) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb new file mode 100644 index 00000000000..bdbcc49f63f --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/select_stmt.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class SelectStmt + include Gitlab::Utils::StrongMemoize + + attr_reader :node, :cte_references, :all_references + + # @param [PgQuery::SelectStmt] node The PgQuery node of the select statement. + # @param [Hash] inherited_cte_references CTE References available to the select statement. + def initialize(node, inherited_cte_references = {}) + @node = node + @cte_references = CommonTableExpressions.references(node, inherited_cte_references) + from_references = Froms.references(node, cte_references) + @all_references = from_references.merge(cte_references) + end + + # returns Set of Types. + # + # STATIC - queries that don't require a database schema lookup. E.g. `SELECT users.id FROM users` + # DYNAMIC - queries that require a database schema lookup. E.g. `SELECT users.* FROM users` + # INVALID - set operator queries that mix static and dynamic queries. + def types + if set_operator? + resolve_set_operator_select_types + else + resolve_normal_select_types + end + end + + private + + # Standard SELECT, not a set operator (UNION/INTERSECT/EXCEPT) + def resolve_normal_select_types + # Cross reference resolved sources with what is requested by the SELECT. + types = Columns.types(self) + + # Mixed dynamic and static queries can be normalized to simply dynamic queries for the purposes of + # detecting mismatched set operator parts. + types.delete(Type::STATIC) if types.include?(Type::DYNAMIC) + + types + end + + # Set operator (UNION/INTERSECT/EXCEPT) + def resolve_set_operator_select_types + types = Set.new + + # Recurse each set operator part as a SELECT statement. + # select statement part => type + set_operator_parts do |part| + types += SelectStmt.new(part, cte_references).types + end + + types << Type::INVALID if types.count > 1 + + types + end + + def set_operator? + !(node.respond_to?(:op) && node.op == :SETOP_NONE) + end + + SET_OPERATOR_PART_LOCATIONS = %i[larg rarg].freeze + private_constant :SET_OPERATOR_PART_LOCATIONS + + def set_operator_parts(&_blk) + return unless node + + yield node if node.op == :SETOP_NONE + yield node.larg if node.larg && node.larg.op == :SETOP_NONE + yield node.rarg if node.rarg && node.rarg.op == :SETOP_NONE + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb new file mode 100644 index 00000000000..99db368efcb --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/targets.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +# Targets refer to SELECT columns but also JOIN fields, etc. +# A target can have a qualifying reference to some other entity like a table or CTE. +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + class Targets + class << self + # Return the reference names used by the given target. + # + # For example: + # `SELECT users.id` would return ['users'] + # `SELECT * FROM users, namespaces` would return ['users', 'namespaces'] + def reference_names(target, select_stmt) + # Parse all targets to determine what is referenced. + fields = fields(target) + case fields.count + when 0 + literal_ref_names(target, select_stmt) + when 1 + unqualified_ref_names(fields, select_stmt) + else + # The target is qualified such as SELECT reference.id + field_ref = fields[fields.count - 2] + [field_ref.string.sval] + end + end + + # True when `SELECT *` + def a_star?(target) + Node.locate_descendant(target, :a_star) + end + + # Null targets are used to produce "polymorphic" query result sets that can be aggregated through a UNION + # without having to worry about mismatched columns. + # + # A null target would be something like: + # SELECT NULL::namespaces FROM namespaces + def null?(target) + target&.val&.type_cast&.arg&.a_const&.isnull + end + + private + + def literal_ref_names(target, select_stmt) + # The target is unqualified and is not part of a column_ref, such as in `SELECT 1`. + # These include targets like literals, functions, and subselects. + sub_select_stmt = subselect_select_stmt(target) + if sub_select_stmt + name = (target.name.presence || "loc_#{target.location}") + # The select is anonymous, so we provide a name. + k = "#{name}_subselect" + # Force parsing of the select. + # We don't care about the static/dynamic nature in this case, but we do need to parse for + # any nested error states. + sub_select = SelectStmt.new(sub_select_stmt, select_stmt.cte_references) + select_stmt.all_references[k] = sub_select.types + [k] + else + # TODO we need to parse function references. Assuming no sources for now. + # https://gitlab.com/gitlab-org/gitlab/-/issues/428102 + [] + end + end + + def unqualified_ref_names(fields, select_stmt) + # The target is unqualified, but is part of a column_ref. + # E.g. `SELECT id FROM namespaces` or `SELECT namespaces FROM namespaces` + + # Otherwise, check all FROM/JOIN/CTE entries. + field = fields[0] + field_sval = field&.string&.sval + if field_sval && select_stmt.all_references.key?(field_sval) + # SELECT some_table_name + [field.string.sval] + else + # SELECT * + # SELECT some_column + select_stmt.all_references.keys + end + end + + def fields(target) + Node.locate_descendants(target, :fields).flatten + end + + def subselect_select_stmt(target) + Node.dig(target, :val, :sub_link, :subselect, :select_stmt) + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb new file mode 100644 index 00000000000..5988f963827 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/prevent_set_operator_mismatch/type.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class PreventSetOperatorMismatch + # An enumerated set of constants that represent the state of the parse. + module Type + STATIC = :static + DYNAMIC = :dynamic + INVALID = :invalid + end + end + end + end +end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table.rb b/lib/gitlab/database/schema_cache_with_renamed_table.rb index 6da76803f7c..e110fc44b7b 100644 --- a/lib/gitlab/database/schema_cache_with_renamed_table.rb +++ b/lib/gitlab/database/schema_cache_with_renamed_table.rb @@ -11,26 +11,26 @@ module Gitlab clear_renamed_tables_cache! end - def clear_data_source_cache!(name) - super(name) + def clear_data_source_cache!(connection, table_name) + super(connection, table_name) clear_renamed_tables_cache! end - def primary_keys(table_name) - super(underlying_table(table_name)) + def primary_keys(connection, table_name) + super(connection, underlying_table(table_name)) end - def columns(table_name) - super(underlying_table(table_name)) + def columns(connection, table_name) + super(connection, underlying_table(table_name)) end - def columns_hash(table_name) - super(underlying_table(table_name)) + def columns_hash(connection, table_name) + super(connection, underlying_table(table_name)) end - def indexes(table_name) - super(underlying_table(table_name)) + def indexes(connection, table_name) + super(connection, underlying_table(table_name)) end private @@ -40,7 +40,7 @@ module Gitlab end def renamed_tables_cache - @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, new_name| + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name| connection.view_exists?(old_name) end end diff --git a/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb new file mode 100644 index 00000000000..acc9bbd0aff --- /dev/null +++ b/lib/gitlab/database/schema_cache_with_renamed_table_legacy.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module Database + # This is a legacy extension targeted at Rails versions prior to 7.1 + # In Rails 7.1, the method parameters have been changed to (connection, table_name) + module SchemaCacheWithRenamedTableLegacy + # Override methods in ActiveRecord::ConnectionAdapters::SchemaCache + + def clear! + super + + clear_renamed_tables_cache! + end + + def clear_data_source_cache!(name) + super(name) + + clear_renamed_tables_cache! + end + + def primary_keys(table_name) + super(underlying_table(table_name)) + end + + def columns(table_name) + super(underlying_table(table_name)) + end + + def columns_hash(table_name) + super(underlying_table(table_name)) + end + + def indexes(table_name) + super(underlying_table(table_name)) + end + + private + + def underlying_table(table_name) + renamed_tables_cache.fetch(table_name, table_name) + end + + def renamed_tables_cache + @renamed_tables ||= Gitlab::Database::TABLES_TO_BE_RENAMED.select do |old_name, _new_name| + connection.view_exists?(old_name) + end + end + + def clear_renamed_tables_cache! + @renamed_tables = nil # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index aa880b709fe..608dea9e3c5 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesLocker - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo gitlab_jh].freeze def initialize(logger: nil, dry_run: false, include_partitions: true) @logger = logger diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index f91146fff3d..5394dee6fec 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesTruncate - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo gitlab_embedding gitlab_jh].freeze def initialize(database_name:, min_batch_size: 5, logger: nil, until_table: nil, dry_run: false) @database_name = database_name diff --git a/lib/gitlab/discussions_diff/file_collection.rb b/lib/gitlab/discussions_diff/file_collection.rb index 60b3a1738f1..3d1f7ab86b3 100644 --- a/lib/gitlab/discussions_diff/file_collection.rb +++ b/lib/gitlab/discussions_diff/file_collection.rb @@ -25,8 +25,9 @@ module Gitlab # # - Highlight cache is written just for uncached diff files # - The cache content is not updated (there's no need to do so) - def load_highlight - ids = highlightable_collection_ids + # - Load only the related diff note ids + def load_highlight(diff_note_ids: nil) + ids = highlightable_collection_ids(diff_note_ids) return if ids.empty? cached_content = read_cache(ids) @@ -47,8 +48,13 @@ module Gitlab private - def highlightable_collection_ids - each.with_object([]) { |file, memo| memo << file.id unless file.resolved_at } + def highlightable_collection_ids(diff_note_ids) + each.with_object([]) do |file, memo| + # We ignore if file is resolved, or not part of the highlight requested notes + next if file.resolved_at || (diff_note_ids.present? && diff_note_ids.exclude?(file.diff_note_id)) + + memo << file.id + end end def read_cache(ids) diff --git a/lib/gitlab/email/handler/service_desk_handler.rb b/lib/gitlab/email/handler/service_desk_handler.rb index ebc4e9c2c8c..e3249b143c8 100644 --- a/lib/gitlab/email/handler/service_desk_handler.rb +++ b/lib/gitlab/email/handler/service_desk_handler.rb @@ -38,7 +38,7 @@ module Gitlab create_issue_or_note if from_address - add_email_participant + add_email_participants send_thank_you_email unless reply_email? end end @@ -215,6 +215,10 @@ module Gitlab end strong_memoize_attr :to_address + def cc_addresses + mail.cc || [] + end + def can_handle_legacy_format? project_path && project_path.include?('/') && !mail_key.include?('+') end @@ -223,11 +227,33 @@ module Gitlab Users::Internal.support_bot end - def add_email_participant + def add_email_participants return if reply_email? && !Feature.enabled?(:issue_email_participants, @issue.project) @issue.issue_email_participants.create(email: from_address) + + add_external_participants_from_cc + end + + def add_external_participants_from_cc + return if project.service_desk_setting.nil? + return unless project.service_desk_setting.add_external_participants_from_cc? + + cc_addresses.each do |email| + next if service_desk_addresses.include?(email) + + @issue.issue_email_participants.create!(email: email) + end + end + + def service_desk_addresses + [ + project.service_desk_incoming_address, + project.service_desk_alias_address, + project.service_desk_custom_address + ].compact end + strong_memoize_attr :service_desk_addresses end end end diff --git a/lib/gitlab/emoji.rb b/lib/gitlab/emoji.rb index 7d47bfe88fe..1a7a2fba2f3 100644 --- a/lib/gitlab/emoji.rb +++ b/lib/gitlab/emoji.rb @@ -6,7 +6,7 @@ module Gitlab # When updating emoji assets increase the version below # and update the version number in `app/assets/javascripts/emoji/index.js` - EMOJI_VERSION = 2 + EMOJI_VERSION = 3 # Return a Pathname to emoji's current versioned folder # diff --git a/lib/gitlab/encrypted_command_base.rb b/lib/gitlab/encrypted_command_base.rb index b35c28b85cd..679d9d8e31a 100644 --- a/lib/gitlab/encrypted_command_base.rb +++ b/lib/gitlab/encrypted_command_base.rb @@ -7,12 +7,12 @@ module Gitlab EDIT_COMMAND_NAME = "base" class << self - def encrypted_secrets + def encrypted_secrets(**args) raise NotImplementedError end - def write(contents) - encrypted = encrypted_secrets + def write(contents, args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) validate_contents(contents) @@ -25,8 +25,8 @@ module Gitlab warn "Couldn't decrypt #{encrypted.content_path}. Perhaps you passed the wrong key?" end - def edit - encrypted = encrypted_secrets + def edit(args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) if ENV["EDITOR"].blank? @@ -58,8 +58,8 @@ module Gitlab temp_file&.unlink end - def show - encrypted = encrypted_secrets + def show(args: {}) + encrypted = encrypted_secrets(**args) return unless validate_config(encrypted) puts encrypted.read.presence || "File '#{encrypted.content_path}' does not exist. Use `gitlab-rake #{self::EDIT_COMMAND_NAME}` to change that." diff --git a/lib/gitlab/encrypted_configuration.rb b/lib/gitlab/encrypted_configuration.rb index 6b64281e631..5ead57e17fd 100644 --- a/lib/gitlab/encrypted_configuration.rb +++ b/lib/gitlab/encrypted_configuration.rb @@ -30,7 +30,7 @@ module Gitlab end def initialize(content_path: nil, base_key: nil, previous_keys: []) - @content_path = Pathname.new(content_path).yield_self { |path| path.symlink? ? path.realpath : path } if content_path + @content_path = Pathname.new(content_path).then { |path| path.symlink? ? path.realpath : path } if content_path @key = self.class.generate_key(base_key) if base_key @previous_keys = previous_keys end diff --git a/lib/gitlab/encrypted_ldap_command.rb b/lib/gitlab/encrypted_ldap_command.rb index 5e1eabe7ec6..442c675f19e 100644 --- a/lib/gitlab/encrypted_ldap_command.rb +++ b/lib/gitlab/encrypted_ldap_command.rb @@ -1,6 +1,5 @@ # frozen_string_literal: true -# rubocop:disable Rails/Output module Gitlab class EncryptedLdapCommand < EncryptedCommandBase DISPLAY_NAME = "LDAP" @@ -21,4 +20,3 @@ module Gitlab end end end -# rubocop:enable Rails/Output diff --git a/lib/gitlab/encrypted_redis_command.rb b/lib/gitlab/encrypted_redis_command.rb new file mode 100644 index 00000000000..608edcdb950 --- /dev/null +++ b/lib/gitlab/encrypted_redis_command.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +# rubocop:disable Rails/Output +module Gitlab + class EncryptedRedisCommand < EncryptedCommandBase + DISPLAY_NAME = "Redis" + EDIT_COMMAND_NAME = "gitlab:redis:secret:edit" + + class << self + def all_redis_instance_class_names + Gitlab::Redis::ALL_CLASSES.map do |c| + normalized_instance_name(c) + end + end + + def normalized_instance_name(instance) + if instance.is_a?(Class) + # Gitlab::Redis::SharedState => sharedstate + instance.name.demodulize.to_s.downcase + else + # Drop all hyphens, underscores, and spaces from the name + # eg.: shared_state => sharedstate + instance.gsub(/[-_ ]/, '').downcase + end + end + + def encrypted_secrets(**args) + if args[:instance_name] + instance_class = Gitlab::Redis::ALL_CLASSES.find do |instance| + normalized_instance_name(instance) == normalized_instance_name(args[:instance_name]) + end + + unless instance_class + error_message = <<~MSG + Specified instance name #{args[:instance_name]} does not exist. + The available instances are #{all_redis_instance_class_names.join(', ')}." + MSG + + raise error_message + end + else + instance_class = Gitlab::Redis::Cache + end + + instance_class.encrypted_secrets + end + + def encrypted_file_template + <<~YAML + # password: '123' + YAML + end + end + end +end +# rubocop:enable Rails/Output diff --git a/lib/gitlab/file_detector.rb b/lib/gitlab/file_detector.rb index 13959f6aa68..ef8f2d4d61b 100644 --- a/lib/gitlab/file_detector.rb +++ b/lib/gitlab/file_detector.rb @@ -21,7 +21,7 @@ module Gitlab # Configuration files gitignore: '.gitignore', - gitlab_ci: '.gitlab-ci.yml', + gitlab_ci: ::Ci::Pipeline::DEFAULT_CONFIG_PATH, route_map: '.gitlab/route-map.yml', # Dependency files diff --git a/lib/gitlab/git/blame.rb b/lib/gitlab/git/blame.rb index 3d2bde6f0a7..e134fb31879 100644 --- a/lib/gitlab/git/blame.rb +++ b/lib/gitlab/git/blame.rb @@ -4,7 +4,6 @@ module Gitlab module Git class Blame include Gitlab::EncodingHelper - include Gitlab::Git::WrapsGitalyErrors attr_reader :lines, :blames, :range @@ -35,11 +34,9 @@ module Gitlab end def fetch_raw_blame - wrapped_gitaly_errors do - @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) - end - # Return empty result when blame range is out-of-range + @repo.gitaly_commit_client.raw_blame(@sha, @path, range: range_spec) rescue ArgumentError + # Return an empty result when the blame range is out-of-range or path is not found "" end diff --git a/lib/gitlab/git/blob.rb b/lib/gitlab/git/blob.rb index ae90291c0a3..3744c81f51d 100644 --- a/lib/gitlab/git/blob.rb +++ b/lib/gitlab/git/blob.rb @@ -230,5 +230,3 @@ module Gitlab end end end - -Gitlab::Git::Blob.singleton_class.prepend Gitlab::Git::RuggedImpl::Blob::ClassMethods diff --git a/lib/gitlab/git/commit.rb b/lib/gitlab/git/commit.rb index 571dde6fcfc..1086ea45a7a 100644 --- a/lib/gitlab/git/commit.rb +++ b/lib/gitlab/git/commit.rb @@ -5,7 +5,6 @@ module Gitlab module Git class Commit include Gitlab::EncodingHelper - prepend Gitlab::Git::RuggedImpl::Commit extend Gitlab::Git::WrapsGitalyErrors include Gitlab::Utils::StrongMemoize @@ -502,5 +501,3 @@ module Gitlab end end end - -Gitlab::Git::Commit.singleton_class.prepend Gitlab::Git::RuggedImpl::Commit::ClassMethods diff --git a/lib/gitlab/git/ref.rb b/lib/gitlab/git/ref.rb index 4a09f866db4..205dd5be35a 100644 --- a/lib/gitlab/git/ref.rb +++ b/lib/gitlab/git/ref.rb @@ -4,7 +4,6 @@ module Gitlab module Git class Ref include Gitlab::EncodingHelper - include Gitlab::Git::RuggedImpl::Ref # Branch or tag name # without "refs/tags|heads" prefix diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index a98cf95edf4..db6e6b4d00b 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -11,7 +11,6 @@ module Gitlab include Gitlab::Git::WrapsGitalyErrors include Gitlab::EncodingHelper include Gitlab::Utils::StrongMemoize - prepend Gitlab::Git::RuggedImpl::Repository SEARCH_CONTEXT_LINES = 3 REV_LIST_COMMIT_LIMIT = 2_000 diff --git a/lib/gitlab/git/rugged_impl/blob.rb b/lib/gitlab/git/rugged_impl/blob.rb deleted file mode 100644 index dc869ff5279..00000000000 --- a/lib/gitlab/git/rugged_impl/blob.rb +++ /dev/null @@ -1,107 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Blob - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - override :tree_entry - def tree_entry(repository, sha, path, limit) - if use_rugged?(repository, :rugged_tree_entry) - execute_rugged_call(:rugged_tree_entry, repository, sha, path, limit) - else - super - end - end - - private - - def rugged_tree_entry(repository, sha, path, limit) - return unless path - - # Strip any leading / characters from the path - path = path.sub(%r{\A/*}, '') - - rugged_commit = repository.lookup(sha) - root_tree = rugged_commit.tree - - blob_entry = find_entry_by_path(repository, root_tree.oid, *path.split('/')) - - return unless blob_entry - - if blob_entry[:type] == :commit - submodule_blob(blob_entry, path, sha) - else - blob = repository.lookup(blob_entry[:oid]) - - if blob - new( - id: blob.oid, - name: blob_entry[:name], - size: blob.size, - # Rugged::Blob#content is expensive; don't call it if we don't have to. - data: limit == 0 ? '' : blob.content(limit), - mode: blob_entry[:filemode].to_s(8), - path: path, - commit_id: sha, - binary: blob.binary? - ) - end - end - rescue Rugged::ReferenceError - nil - end - - # Recursive search of blob id by path - # - # Ex. - # blog/ # oid: 1a - # app/ # oid: 2a - # models/ # oid: 3a - # file.rb # oid: 4a - # - # - # Blob.find_entry_by_path(repo, '1a', 'blog', 'app', 'file.rb') # => '4a' - # - def find_entry_by_path(repository, root_id, *path_parts) - root_tree = repository.lookup(root_id) - - entry = root_tree.find do |entry| - entry[:name] == path_parts[0] - end - - return unless entry - - if path_parts.size > 1 - return unless entry[:type] == :tree - - path_parts.shift - find_entry_by_path(repository, entry[:oid], *path_parts) - else - [:blob, :commit].include?(entry[:type]) ? entry : nil - end - end - - def submodule_blob(blob_entry, path, sha) - new( - id: blob_entry[:oid], - name: blob_entry[:name], - size: 0, - data: '', - path: path, - commit_id: sha - ) - end - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/commit.rb b/lib/gitlab/git/rugged_impl/commit.rb deleted file mode 100644 index cf547414b0d..00000000000 --- a/lib/gitlab/git/rugged_impl/commit.rb +++ /dev/null @@ -1,115 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Git - module RuggedImpl - module Commit - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - def rugged_find(repo, commit_id) - obj = repo.rev_parse_target(commit_id) - - obj.is_a?(::Rugged::Commit) ? obj : nil - rescue ::Rugged::Error - nil - end - - # This needs to return an array of Gitlab::Git:Commit objects - # instead of Rugged::Commit objects to ensure upstream models - # operate on a consistent interface. Unlike - # Gitlab::Git::Commit.find, Gitlab::Git::Commit.batch_by_oid - # doesn't attempt to decorate the result. - def rugged_batch_by_oid(repo, oids) - oids.map { |oid| rugged_find(repo, oid) } - .compact - .map { |commit| decorate(repo, commit) } - # Match Gitaly's list_commits_by_oid behavior - rescue ::Gitlab::Git::Repository::NoRepository - [] - end - - override :find_commit - def find_commit(repo, commit_id) - if use_rugged?(repo, :rugged_find_commit) - execute_rugged_call(:rugged_find, repo, commit_id) - else - super - end - end - - override :batch_by_oid - def batch_by_oid(repo, oids) - if use_rugged?(repo, :rugged_list_commits_by_oid) - execute_rugged_call(:rugged_batch_by_oid, repo, oids) - else - super - end - end - end - - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - override :init_commit - def init_commit(raw_commit) - case raw_commit - when ::Rugged::Commit - init_from_rugged(raw_commit) - else - super - end - end - - override :commit_tree_entry - def commit_tree_entry(path) - if use_rugged?(@repository, :rugged_commit_tree_entry) - execute_rugged_call(:rugged_tree_entry, path) - else - super - end - end - - # Is this the same as Blob.find_entry_by_path ? - def rugged_tree_entry(path) - rugged_commit.tree.path(path) - rescue Rugged::TreeError - nil - end - - def rugged_commit - @rugged_commit ||= if raw_commit.is_a?(Rugged::Commit) - raw_commit - else - @repository.rev_parse_target(id) - end - end - - def init_from_rugged(commit) - author = commit.author - committer = commit.committer - - @raw_commit = commit - @id = commit.oid - @message = commit.message - @authored_date = author[:time] - @committed_date = committer[:time] - @author_name = author[:name] - @author_email = author[:email] - @committer_name = committer[:name] - @committer_email = committer[:email] - @parent_ids = commit.parents.map(&:oid) - @trailers = Hash[commit.trailers] - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/ref.rb b/lib/gitlab/git/rugged_impl/ref.rb deleted file mode 100644 index b553e82dc47..00000000000 --- a/lib/gitlab/git/rugged_impl/ref.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Ref - def self.dereference_object(object) - object = object.target while object.is_a?(::Rugged::Tag::Annotation) - - object - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/repository.rb b/lib/gitlab/git/rugged_impl/repository.rb deleted file mode 100644 index cd4eefa158e..00000000000 --- a/lib/gitlab/git/rugged_impl/repository.rb +++ /dev/null @@ -1,79 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -# rubocop:disable Gitlab/ModuleWithInstanceVariables -module Gitlab - module Git - module RuggedImpl - module Repository - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - FEATURE_FLAGS = %i[rugged_find_commit rugged_tree_entries rugged_tree_entry rugged_commit_is_ancestor rugged_commit_tree_entry rugged_list_commits_by_oid].freeze - - def alternate_object_directories - relative_object_directories.map { |d| File.join(path, d) } - end - - ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES = %w[ - GIT_OBJECT_DIRECTORY_RELATIVE - GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE - ].freeze - - def relative_object_directories - Gitlab::Git::HookEnv.all(gl_repository).values_at(*ALLOWED_OBJECT_RELATIVE_DIRECTORIES_VARIABLES).flatten.compact - end - - def rugged - @rugged ||= ::Rugged::Repository.new(path, alternates: alternate_object_directories) - rescue ::Rugged::RepositoryError, ::Rugged::OSError - raise ::Gitlab::Git::Repository::NoRepository, 'no repository for such path' - end - - def cleanup - @rugged&.close - end - - # Return the object that +revspec+ points to. If +revspec+ is an - # annotated tag, then return the tag's target instead. - def rev_parse_target(revspec) - obj = rugged.rev_parse(revspec) - Ref.dereference_object(obj) - end - - override :ancestor? - def ancestor?(from, to) - if use_rugged?(self, :rugged_commit_is_ancestor) - execute_rugged_call(:rugged_is_ancestor?, from, to) - else - super - end - end - - def rugged_is_ancestor?(ancestor_id, descendant_id) - return false if ancestor_id.nil? || descendant_id.nil? - - rugged_merge_base(ancestor_id, descendant_id) == ancestor_id - rescue Rugged::OdbError - false - end - - def rugged_merge_base(from, to) - rugged.merge_base(from, to) - rescue Rugged::ReferenceError - nil - end - - # Lookup for rugged object by oid or ref name - def lookup(oid_or_ref_name) - rev_parse_target(oid_or_ref_name) - end - end - end - end -end -# rubocop:enable Gitlab/ModuleWithInstanceVariables diff --git a/lib/gitlab/git/rugged_impl/tree.rb b/lib/gitlab/git/rugged_impl/tree.rb deleted file mode 100644 index bc3ff01e1e2..00000000000 --- a/lib/gitlab/git/rugged_impl/tree.rb +++ /dev/null @@ -1,147 +0,0 @@ -# frozen_string_literal: true - -# NOTE: This code is legacy. Do not add/modify code here unless you have -# discussed with the Gitaly team. See -# https://docs.gitlab.com/ee/development/gitaly.html#legacy-rugged-code -# for more details. - -module Gitlab - module Git - module RuggedImpl - module Tree - module ClassMethods - extend ::Gitlab::Utils::Override - include Gitlab::Git::RuggedImpl::UseRugged - - TREE_SORT_ORDER = { tree: 0, blob: 1, commit: 2 }.freeze - - override :tree_entries - def tree_entries(repository, sha, path, recursive, skip_flat_paths, rescue_not_found, pagination_params = nil) - if use_rugged?(repository, :rugged_tree_entries) - entries = execute_rugged_call( - :tree_entries_with_flat_path_from_rugged, repository, sha, path, recursive, skip_flat_paths) - - if pagination_params - paginated_response(entries, pagination_params[:limit], pagination_params[:page_token].to_s) - else - [entries, nil] - end - else - super - end - end - - # Rugged version of TreePagination in Go: https://gitlab.com/gitlab-org/gitaly/-/merge_requests/3611 - def paginated_response(entries, limit, token) - total_entries = entries.count - - return [[], nil] if limit == 0 || limit.blank? - - entries = Gitlab::Utils.stable_sort_by(entries) { |x| TREE_SORT_ORDER[x.type] } - - if token.blank? - index = 0 - else - index = entries.index { |entry| entry.id == token } - - raise Gitlab::Git::CommandError, "could not find starting OID: #{token}" if index.nil? - - index += 1 - end - - return [entries[index..], nil] if limit < 0 - - last_index = index + limit - result = entries[index...last_index] - - if last_index < total_entries - cursor = Gitaly::PaginationCursor.new(next_cursor: result.last.id) - end - - [result, cursor] - end - - def tree_entries_with_flat_path_from_rugged(repository, sha, path, recursive, skip_flat_paths) - tree_entries_from_rugged(repository, sha, path, recursive).tap do |entries| - # This was an optimization to reduce N+1 queries for Gitaly - # (https://gitlab.com/gitlab-org/gitaly/issues/530). - rugged_populate_flat_path(repository, sha, path, entries) unless skip_flat_paths - end - end - - def tree_entries_from_rugged(repository, sha, path, recursive) - current_path_entries = get_tree_entries_from_rugged(repository, sha, path) - ordered_entries = [] - - current_path_entries.each do |entry| - ordered_entries << entry - - if recursive && entry.dir? - ordered_entries.concat(tree_entries_from_rugged(repository, sha, entry.path, true)) - end - end - - ordered_entries - end - - def rugged_populate_flat_path(repository, sha, path, entries) - entries.each do |entry| - entry.flat_path = entry.path - - next unless entry.dir? - - entry.flat_path = - if path - File.join(path, rugged_flatten_tree(repository, sha, entry, path)) - else - rugged_flatten_tree(repository, sha, entry, path) - end - end - end - - # Returns the relative path of the first subdir that doesn't have only one directory descendant - def rugged_flatten_tree(repository, sha, tree, root_path) - subtree = tree_entries_from_rugged(repository, sha, tree.path, false) - - if subtree.count == 1 && subtree.first.dir? - File.join(tree.name, rugged_flatten_tree(repository, sha, subtree.first, root_path)) - else - tree.name - end - end - - def get_tree_entries_from_rugged(repository, sha, path) - commit = repository.lookup(sha) - root_tree = commit.tree - - tree = if path - id = find_id_by_path(repository, root_tree.oid, path) - if id - repository.lookup(id) - else - [] - end - else - root_tree - end - - tree.map do |entry| - current_path = path ? File.join(path, entry[:name]) : entry[:name] - - new( - id: entry[:oid], - name: entry[:name], - type: entry[:type], - mode: entry[:filemode].to_s(8), - path: current_path, - commit_id: sha - ) - end - rescue Rugged::ReferenceError - [] - end - end - end - end - end -end diff --git a/lib/gitlab/git/rugged_impl/use_rugged.rb b/lib/gitlab/git/rugged_impl/use_rugged.rb deleted file mode 100644 index 57cced97d02..00000000000 --- a/lib/gitlab/git/rugged_impl/use_rugged.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Git - module RuggedImpl - module UseRugged - def use_rugged?(_, _) - false - end - - def execute_rugged_call(method_name, *args) - Gitlab::GitalyClient::StorageSettings.allow_disk_access do - start = Gitlab::Metrics::System.monotonic_time - - result = send(method_name, *args) # rubocop:disable GitlabSecurity/PublicSend - - duration = Gitlab::Metrics::System.monotonic_time - start - - if Gitlab::RuggedInstrumentation.active? - Gitlab::RuggedInstrumentation.increment_query_count - Gitlab::RuggedInstrumentation.add_query_time(duration) - - Gitlab::RuggedInstrumentation.add_call_details( - feature: method_name, - args: args, - duration: duration, - backtrace: Gitlab::BacktraceCleaner.clean_backtrace(caller)) - end - - result - end - end - - def running_puma_with_multiple_threads? - return false unless Gitlab::Runtime.puma? - - ::Puma.respond_to?(:cli_config) && ::Puma.cli_config.options[:max_threads] > 1 - end - - def rugged_feature_keys - Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS - end - - def rugged_enabled_through_feature_flag? - false - end - end - end - end -end diff --git a/lib/gitlab/git/tree.rb b/lib/gitlab/git/tree.rb index 6e97e412b91..4747ab55c63 100644 --- a/lib/gitlab/git/tree.rb +++ b/lib/gitlab/git/tree.rb @@ -12,9 +12,6 @@ module Gitlab class << self # Get list of tree objects # for repository based on commit sha and path - # Uses rugged for raw objects - # - # Gitaly migration: https://gitlab.com/gitlab-org/gitaly/issues/320 def where( repository, sha, path = nil, recursive = false, skip_flat_paths = true, rescue_not_found = true, pagination_params = nil) @@ -110,5 +107,3 @@ module Gitlab end end end - -Gitlab::Git::Tree.singleton_class.prepend Gitlab::Git::RuggedImpl::Tree::ClassMethods diff --git a/lib/gitlab/git_access.rb b/lib/gitlab/git_access.rb index 45283d51b1b..72016aa1183 100644 --- a/lib/gitlab/git_access.rb +++ b/lib/gitlab/git_access.rb @@ -101,7 +101,7 @@ module Gitlab end def guest_can_download? - Guest.can?(download_ability, container) + ::Users::Anonymous.can?(download_ability, container) end def deploy_key_can_download_code? @@ -395,7 +395,7 @@ module Gitlab user.can?(:read_project, project) elsif ci? false - end || Guest.can?(:read_project, project) + end || ::Users::Anonymous.can?(:read_project, project) end def http? diff --git a/lib/gitlab/git_access_project.rb b/lib/gitlab/git_access_project.rb index 732e0e14257..b007a957348 100644 --- a/lib/gitlab/git_access_project.rb +++ b/lib/gitlab/git_access_project.rb @@ -47,7 +47,7 @@ module Gitlab end def repository_path_match - strong_memoize(:repository_path_match) { repository_path.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } + strong_memoize(:repository_path_match) { repository_path&.match(Gitlab::PathRegex.full_project_git_path_regex) || {} } end def ensure_project_on_push! diff --git a/lib/gitlab/git_audit_event.rb b/lib/gitlab/git_audit_event.rb deleted file mode 100644 index b8365bdf41f..00000000000 --- a/lib/gitlab/git_audit_event.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - class GitAuditEvent # rubocop:disable Gitlab/NamespacedClass - attr_reader :project, :user, :author - - def initialize(player, project) - @project = project - @author = player.is_a?(::API::Support::GitAccessActor) ? player.deploy_key_or_user : player - @user = player.is_a?(::API::Support::GitAccessActor) ? player.user : player - end - - def send_audit_event(msg) - return if user.blank? || project.blank? - - audit_context = { - name: 'repository_git_operation', - stream_only: true, - author: author, - scope: project, - target: project, - message: msg - } - - ::Gitlab::Audit::Auditor.audit(audit_context) - end - end -end diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 5ec58fc4f44..da38c11ebca 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -328,6 +328,8 @@ module Gitlab 'client_name' => CLIENT_NAME } + relative_path = fetch_relative_path + context_data = Gitlab::ApplicationContext.current feature_stack = Thread.current[:gitaly_feature_stack] @@ -339,6 +341,7 @@ module Gitlab metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) + metadata['relative-path-bin'] = relative_path if relative_path metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors)) metadata.merge!(route_to_primary) @@ -348,6 +351,17 @@ module Gitlab { metadata: metadata, deadline: deadline_info[:deadline] } end + # The GitLab `internal/allowed/` API sets the :gitlab_git_relative_path + # variable. This provides the repository relative path which can be used to + # locate snapshot repositories in Gitaly which act as a quarantine repository + # until a transaction is committed. + def self.fetch_relative_path + return unless Gitlab::SafeRequestStore.active? + return if Gitlab::SafeRequestStore[:gitlab_git_relative_path].blank? + + Gitlab::SafeRequestStore.fetch(:gitlab_git_relative_path) + end + # Gitlab::Git::HookEnv will set the :gitlab_git_env variable in case we're # running in the context of a Gitaly hook call, which may make use of # quarantined object directories. We thus need to pass along the path of diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 1ef5b0f96c2..3949e8e6416 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -418,6 +418,15 @@ module Gitlab response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error.try(:error) + when :out_of_range, :path_not_found + raise ArgumentError, e.details + else + raise e + end end def find_commit(revision) diff --git a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb index b1278e3bfac..a6912547ce9 100644 --- a/lib/gitlab/gitaly_client/conflict_files_stitcher.rb +++ b/lib/gitlab/gitaly_client/conflict_files_stitcher.rb @@ -43,11 +43,15 @@ module Gitlab def conflict_from_gitaly_file_header(header) { - ancestor: { path: header.ancestor_path }, - ours: { path: header.our_path, mode: header.our_mode }, - theirs: { path: header.their_path } + ancestor: { path: encode_path(header.ancestor_path) }, + ours: { path: encode_path(header.our_path), mode: header.our_mode }, + theirs: { path: encode_path(header.their_path) } } end + + def encode_path(path) + Gitlab::EncodingHelper.encode_utf8(path) + end end end end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index d92bf5263f1..457380615f7 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -136,10 +136,13 @@ module Gitlab response.base.presence end - def fork_repository(source_repository) + def fork_repository(source_repository, branch = nil) + revision = branch.present? ? "refs/heads/#{branch}" : "" + request = Gitaly::CreateForkRequest.new( repository: @gitaly_repo, - source_repository: source_repository.gitaly_repository + source_repository: source_repository.gitaly_repository, + revision: revision ) gitaly_client_call( diff --git a/lib/gitlab/gitaly_client/storage_settings.rb b/lib/gitlab/gitaly_client/storage_settings.rb index 4cc0269673f..adf0c811274 100644 --- a/lib/gitlab/gitaly_client/storage_settings.rb +++ b/lib/gitlab/gitaly_client/storage_settings.rb @@ -31,19 +31,11 @@ module Gitlab end def self.disk_access_denied? - return false if rugged_enabled? - !temporarily_allowed?(ALLOW_KEY) rescue StandardError false # Err on the side of caution, don't break gitlab for people end - def self.rugged_enabled? - Gitlab::Git::RuggedImpl::Repository::FEATURE_FLAGS.any? do |flag| - Feature.enabled?(flag) - end - end - def initialize(storage) raise InvalidConfigurationError, "expected a Hash, got a #{storage.class.name}" unless storage.is_a?(Hash) raise InvalidConfigurationError, INVALID_STORAGE_MESSAGE unless storage.has_key?('path') diff --git a/lib/gitlab/github_import/attachments_downloader.rb b/lib/gitlab/github_import/attachments_downloader.rb index 4db55a6aabb..df9c6c8342d 100644 --- a/lib/gitlab/github_import/attachments_downloader.rb +++ b/lib/gitlab/github_import/attachments_downloader.rb @@ -29,8 +29,8 @@ module Gitlab validate_content_length validate_filepath - redirection_url = get_assets_download_redirection_url - file = download_from(redirection_url) + download_url = get_assets_download_redirection_url + file = download_from(download_url) validate_symlink file @@ -60,16 +60,16 @@ module Gitlab options[:follow_redirects] = false response = Gitlab::HTTP.perform_request(Net::HTTP::Get, file_url, options) - raise_error("expected a redirect response, got #{response.code}") unless response.redirection? - redirection_url = response.headers[:location] - filename = URI.parse(redirection_url).path + download_url = if response.redirection? + response.headers[:location] + else + file_url + end - unless Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| filename.ends_with?(type) } - raise UnsupportedAttachmentError - end + file_type_valid?(URI.parse(download_url).path) - redirection_url + download_url end def github_assets_url_regex @@ -89,6 +89,12 @@ module Gitlab File.join(dir, filename) end end + + def file_type_valid?(file_url) + return if Gitlab::GithubImport::Markdown::Attachment::MEDIA_TYPES.any? { |type| file_url.ends_with?(type) } + + raise UnsupportedAttachmentError + end end end end diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 5a0ae680ab8..33e74c90115 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -182,12 +182,12 @@ module Gitlab request_count_counter.increment - raise_or_wait_for_rate_limit unless requests_remaining? + raise_or_wait_for_rate_limit('Internal threshold reached') unless requests_remaining? begin with_retry { yield } - rescue ::Octokit::TooManyRequests - raise_or_wait_for_rate_limit + rescue ::Octokit::TooManyRequests => e + raise_or_wait_for_rate_limit(e.response_body) # This retry will only happen when running in sequential mode as we'll # raise an error in parallel mode. @@ -213,11 +213,11 @@ module Gitlab octokit.rate_limit.limit end - def raise_or_wait_for_rate_limit + def raise_or_wait_for_rate_limit(message) rate_limit_counter.increment if parallel? - raise RateLimitError + raise RateLimitError, message else sleep(rate_limit_resets_in) end diff --git a/lib/gitlab/github_import/issuable_finder.rb b/lib/gitlab/github_import/issuable_finder.rb index b960df581e4..0780ba6119f 100644 --- a/lib/gitlab/github_import/issuable_finder.rb +++ b/lib/gitlab/github_import/issuable_finder.rb @@ -11,6 +11,7 @@ module Gitlab # The base cache key to use for storing/retrieving issuable IDs. CACHE_KEY = 'github-import/issuable-finder/%{project}/%{type}/%{iid}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project`. # object - The object to look up or set a database ID for. @@ -23,9 +24,18 @@ module Gitlab # # This method will return `nil` if no ID could be found. def database_id - val = Gitlab::Cache::Import::Caching.read(cache_key, timeout: timeout) + val = Gitlab::Cache::Import::Caching.read_integer(cache_key, timeout: timeout) - val.to_i if val.present? + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = cache_key_type.safe_constantize&.find_by(project_id: project.id, iid: cache_key_iid)&.id || + CACHE_OBJECT_NOT_FOUND + + cache_database_id(object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # Associates the given database ID with the current object. diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index 52b211c92d6..077a27df16c 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -15,7 +15,7 @@ module Gitlab def calculate_job_delay(job_index) multiplier = (job_index / parallel_import_batch[:size]) - (multiplier * parallel_import_batch[:delay]) + 1.second + (multiplier * parallel_import_batch[:delay]).to_i + 1 end end end diff --git a/lib/gitlab/github_import/label_finder.rb b/lib/gitlab/github_import/label_finder.rb index 39e669dbba4..d0bbd2bc7cf 100644 --- a/lib/gitlab/github_import/label_finder.rb +++ b/lib/gitlab/github_import/label_finder.rb @@ -7,6 +7,7 @@ module Gitlab # The base cache key to use for storing/retrieving label IDs. CACHE_KEY = 'github-import/label-finder/%{project}/%{name}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project`. def initialize(project) @@ -15,7 +16,18 @@ module Gitlab # Returns the label ID for the given name. def id_for(name) - Gitlab::Cache::Import::Caching.read_integer(cache_key_for(name)) + cache_key = cache_key_for(name) + val = Gitlab::Cache::Import::Caching.read_integer(cache_key) + + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = project.labels.with_title(name).pick(:id) || CACHE_OBJECT_NOT_FOUND + + Gitlab::Cache::Import::Caching.write(cache_key, object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # rubocop: disable CodeReuse/ActiveRecord @@ -32,7 +44,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(name) - CACHE_KEY % { project: project.id, name: name } + format(CACHE_KEY, project: project.id, name: name) end end end diff --git a/lib/gitlab/github_import/milestone_finder.rb b/lib/gitlab/github_import/milestone_finder.rb index d9290e36ea1..dcb679fda6d 100644 --- a/lib/gitlab/github_import/milestone_finder.rb +++ b/lib/gitlab/github_import/milestone_finder.rb @@ -7,6 +7,7 @@ module Gitlab # The base cache key to use for storing/retrieving milestone IDs. CACHE_KEY = 'github-import/milestone-finder/%{project}/%{iid}' + CACHE_OBJECT_NOT_FOUND = -1 # project - An instance of `Project` def initialize(project) @@ -18,7 +19,20 @@ module Gitlab def id_for(issuable) return unless issuable.milestone_number - Gitlab::Cache::Import::Caching.read_integer(cache_key_for(issuable.milestone_number)) + milestone_iid = issuable.milestone_number + cache_key = cache_key_for(milestone_iid) + + val = Gitlab::Cache::Import::Caching.read_integer(cache_key) + + return val if Feature.disabled?(:import_fallback_to_db_empty_cache, project) + + return if val == CACHE_OBJECT_NOT_FOUND + return val if val.present? + + object_id = project.milestones.by_iid(milestone_iid).pick(:id) || CACHE_OBJECT_NOT_FOUND + + Gitlab::Cache::Import::Caching.write(cache_key, object_id) + object_id == CACHE_OBJECT_NOT_FOUND ? nil : object_id end # rubocop: disable CodeReuse/ActiveRecord @@ -35,7 +49,7 @@ module Gitlab # rubocop: enable CodeReuse/ActiveRecord def cache_key_for(iid) - CACHE_KEY % { project: project.id, iid: iid } + format(CACHE_KEY, project: project.id, iid: iid) end end end diff --git a/lib/gitlab/github_import/object_counter.rb b/lib/gitlab/github_import/object_counter.rb index 88e91800cee..5618cfc6044 100644 --- a/lib/gitlab/github_import/object_counter.rb +++ b/lib/gitlab/github_import/object_counter.rb @@ -52,7 +52,7 @@ module Gitlab .sort .each do |counter| object_type = counter.split('/').last - result[operation][object_type] = CACHING.read_integer(counter) || 0 + result[operation][object_type] = CACHING.read_integer(counter, timeout: IMPORT_CACHING_TIMEOUT) || 0 end end end diff --git a/lib/gitlab/github_import/parallel_scheduling.rb b/lib/gitlab/github_import/parallel_scheduling.rb index cccd99f48b1..ce93b5203df 100644 --- a/lib/gitlab/github_import/parallel_scheduling.rb +++ b/lib/gitlab/github_import/parallel_scheduling.rb @@ -6,7 +6,7 @@ module Gitlab include JobDelayCalculator attr_reader :project, :client, :page_counter, :already_imported_cache_key, - :job_waiter_cache_key, :job_waiter_remaining_cache_key + :job_waiter_cache_key, :job_waiter_remaining_cache_key # The base cache key to use for tracking already imported objects. ALREADY_IMPORTED_CACHE_KEY = @@ -26,12 +26,11 @@ module Gitlab @client = client @parallel = parallel @page_counter = PageCounter.new(project, collection_method) - @already_imported_cache_key = ALREADY_IMPORTED_CACHE_KEY % - { project: project.id, collection: collection_method } - @job_waiter_cache_key = JOB_WAITER_CACHE_KEY % - { project: project.id, collection: collection_method } - @job_waiter_remaining_cache_key = JOB_WAITER_REMAINING_CACHE_KEY % - { project: project.id, collection: collection_method } + @already_imported_cache_key = format(ALREADY_IMPORTED_CACHE_KEY, project: project.id, + collection: collection_method) + @job_waiter_cache_key = format(JOB_WAITER_CACHE_KEY, project: project.id, collection: collection_method) + @job_waiter_remaining_cache_key = format(JOB_WAITER_REMAINING_CACHE_KEY, project: project.id, + collection: collection_method) end def parallel? @@ -57,7 +56,8 @@ module Gitlab # still scheduling duplicates while. Since all work has already been # completed those jobs will just cycle through any remaining pages while # not scheduling anything. - Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) + Gitlab::Cache::Import::Caching.expire(already_imported_cache_key, + Gitlab::Cache::Import::Caching::SHORTER_TIMEOUT) info(project.id, message: "importer finished") retval @@ -97,7 +97,7 @@ module Gitlab repr = object_representation(object) job_delay = calculate_job_delay(enqueued_job_counter) - sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash, job_waiter.key) + sidekiq_worker_class.perform_in(job_delay, project.id, repr.to_hash.deep_stringify_keys, job_waiter.key.to_s) enqueued_job_counter += 1 job_waiter.jobs_remaining = Gitlab::Cache::Import::Caching.increment(job_waiter_remaining_cache_key) diff --git a/lib/gitlab/github_import/representation/to_hash.rb b/lib/gitlab/github_import/representation/to_hash.rb index 4a0f36ab8f0..54faa51a787 100644 --- a/lib/gitlab/github_import/representation/to_hash.rb +++ b/lib/gitlab/github_import/representation/to_hash.rb @@ -16,11 +16,15 @@ module Gitlab hash end + # This method allow objects to be safely passed directly to Sidekiq without errors. + # It returns JSON datatypes: string, integer, float, boolean, null(nil), array and hash. def convert_value_for_to_hash(value) if value.is_a?(Array) value.map { |v| convert_value_for_to_hash(v) } elsif value.respond_to?(:to_hash) value.to_hash + elsif value.respond_to?(:strftime) || value.is_a?(Symbol) + value.to_s else value end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index e057b4bb6f1..59813e4f5a0 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -50,6 +50,7 @@ module Gitlab gon.suggested_label_colors = LabelsHelper.suggested_colors gon.first_day_of_week = current_user&.first_day_of_week || Gitlab::CurrentSettings.first_day_of_week gon.time_display_relative = true + gon.time_display_format = 0 gon.ee = Gitlab.ee? gon.jh = Gitlab.jh? gon.dot_com = Gitlab.com? @@ -67,6 +68,7 @@ module Gitlab gon.current_user_fullname = current_user.name gon.current_user_avatar_url = current_user.avatar_url gon.time_display_relative = current_user.time_display_relative + gon.time_display_format = current_user.time_display_format end # Initialize gon.features with any flags that should be @@ -75,7 +77,6 @@ module Gitlab push_frontend_feature_flag(:security_auto_fix) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:vscode_web_ide, current_user) - push_frontend_feature_flag(:unbatch_graphql_queries, current_user) # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/399248 push_frontend_feature_flag(:remove_monitor_metrics) push_frontend_feature_flag(:custom_emoji) diff --git a/lib/gitlab/graphql/tracers/timer_tracer.rb b/lib/gitlab/graphql/tracers/timer_tracer.rb index 8e058621110..2cf06086a3c 100644 --- a/lib/gitlab/graphql/tracers/timer_tracer.rb +++ b/lib/gitlab/graphql/tracers/timer_tracer.rb @@ -15,11 +15,11 @@ module Gitlab end def trace(key, data) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = ::Gitlab::Metrics::System.monotonic_time yield ensure - data[:duration_s] = Gitlab::Metrics::System.monotonic_time - start_time + data[:duration_s] = ::Gitlab::Metrics::System.monotonic_time - start_time end end end diff --git a/lib/gitlab/group_search_results.rb b/lib/gitlab/group_search_results.rb index 8ca88859b22..6fe7a0030f0 100644 --- a/lib/gitlab/group_search_results.rb +++ b/lib/gitlab/group_search_results.rb @@ -13,7 +13,7 @@ module Gitlab # rubocop:disable CodeReuse/ActiveRecord def users groups = group.self_and_hierarchy_intersecting_with_user_groups(current_user) - groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/417455") + groups = groups.allow_cross_joins_across_databases(url: "https://gitlab.com/gitlab-org/gitlab/-/issues/427108") members = GroupMember.where(group: groups).non_invite users = super diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index 08d44184bb6..720f8748cba 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -# Detect user based on identifier like +# Detect user or keys based on identifier like # key-13 or user-36 module Gitlab module Identifier @@ -35,6 +35,13 @@ module Gitlab end end + # Tries to identify a deploy key using a SSH key identifier (e.g. "key-123"). + def identify_using_deploy_key(identifier) + key_id = identifier.gsub("key-", "") + + DeployKey.find_by_id(key_id) + end + def identify_with_cache(category, key) if identification_cache[category].key?(key) identification_cache[category][key] diff --git a/lib/gitlab/import_export/command_line_util.rb b/lib/gitlab/import_export/command_line_util.rb index ea91b01afdb..523df1f9d5e 100644 --- a/lib/gitlab/import_export/command_line_util.rb +++ b/lib/gitlab/import_export/command_line_util.rb @@ -40,13 +40,12 @@ module Gitlab cmd = %W[gzip #{filepath}] cmd << "-#{options}" if options - _, status = Gitlab::Popen.popen(cmd) + output, status = Gitlab::Popen.popen(cmd) - if status == 0 - status - else - raise Gitlab::ImportExport::Error.file_compression_error - end + return status if status == 0 + + message = cmd_error_message(output, status) + raise Gitlab::ImportExport::Error.file_compression_error(message) end def mkdir_p(path) @@ -104,9 +103,7 @@ module Gitlab return true if status == 0 - output = output&.strip - message = "command exited with error code #{status}" - message += ": #{output}" if output.present? + message = cmd_error_message(output, status) if @shared.respond_to?(:error) @shared.error(Gitlab::ImportExport::Error.new(message)) @@ -149,6 +146,12 @@ module Gitlab FileUtils.remove_dir(dir) raise end + + def cmd_error_message(output, status) + message = "Command exited with error code #{status}" + message << ": #{output.strip}" unless output.blank? + message + end end end end diff --git a/lib/gitlab/import_export/error.rb b/lib/gitlab/import_export/error.rb index fa179f584eb..9b8e6374b5a 100644 --- a/lib/gitlab/import_export/error.rb +++ b/lib/gitlab/import_export/error.rb @@ -14,8 +14,8 @@ module Gitlab self.new('Unknown object type') end - def self.file_compression_error - self.new('File compression/decompression failed') + def self.file_compression_error(error) + self.new(format('File compression or decompression failed. %{error}', error: error)) end def self.incompatible_import_file_error diff --git a/lib/gitlab/import_export/project/sample/date_calculator.rb b/lib/gitlab/import_export/project/sample/date_calculator.rb index 543fd25d883..0cb0eb32a23 100644 --- a/lib/gitlab/import_export/project/sample/date_calculator.rb +++ b/lib/gitlab/import_export/project/sample/date_calculator.rb @@ -25,7 +25,7 @@ module Gitlab end def calculate_by_closest_date_to_average(date) - return date unless closest_date_to_average && closest_date_to_average < Time.current + return date unless closest_date_to_average && closest_date_to_average.past? date + (Time.current - closest_date_to_average).seconds end diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index e39bbb36680..88991495a10 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -90,7 +90,7 @@ module Gitlab result = ::Gitlab::Instrumentation::RedisClusterValidator.validate(commands) return true if result.nil? - if !result[:valid] && !result[:allowed] && (Rails.env.development? || Rails.env.test?) + if !result[:valid] && !result[:allowed] && raise_cross_slot_validation_errors? raise RedisClusterValidator::CrossSlotError, "Redis command #{result[:command_name]} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end @@ -189,6 +189,10 @@ module Gitlab redirection_type, _, target_node_key = err_msg.split { redirection_type: redirection_type, target_node_key: target_node_key } end + + def raise_cross_slot_validation_errors? + Rails.env.development? || Rails.env.test? + end end end end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 20ba1ab82a7..5934204bd0f 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -31,7 +31,7 @@ module Gitlab private def instrument_call(commands, pipelined = false) - start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined + start = ::Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) instrumentation_class.instance_count_pipelined_request(commands.size) if pipelined @@ -50,7 +50,7 @@ module Gitlab instrumentation_class.log_exception(ex) raise ex ensure - duration = Gitlab::Metrics::System.monotonic_time - start + duration = ::Gitlab::Metrics::System.monotonic_time - start unless exclude_from_apdex?(commands) commands.each { instrumentation_class.instance_observe_duration(duration / commands.size) } diff --git a/lib/gitlab/instrumentation_helper.rb b/lib/gitlab/instrumentation_helper.rb index 2a3c4db5ffa..49078a7ccd0 100644 --- a/lib/gitlab/instrumentation_helper.rb +++ b/lib/gitlab/instrumentation_helper.rb @@ -12,7 +12,6 @@ module Gitlab def add_instrumentation_data(payload) instrument_gitaly(payload) - instrument_rugged(payload) instrument_redis(payload) instrument_elasticsearch(payload) instrument_zoekt(payload) @@ -40,15 +39,6 @@ module Gitlab payload[:gitaly_duration_s] = Gitlab::GitalyClient.query_time end - def instrument_rugged(payload) - rugged_calls = Gitlab::RuggedInstrumentation.query_count - - return if rugged_calls == 0 - - payload[:rugged_calls] = rugged_calls - payload[:rugged_duration_s] = Gitlab::RuggedInstrumentation.query_time - end - def instrument_redis(payload) payload.merge! ::Gitlab::Instrumentation::Redis.payload end diff --git a/lib/gitlab/internal_events.rb b/lib/gitlab/internal_events.rb index 2790bc8ee24..e2e4ea75dbf 100644 --- a/lib/gitlab/internal_events.rb +++ b/lib/gitlab/internal_events.rb @@ -23,8 +23,6 @@ module Gitlab private def increase_total_counter(event_name) - return unless ::ServicePing::ServicePingSettings.enabled? - redis_counter_key = Gitlab::Usage::Metrics::Instrumentations::TotalCountMetric.redis_key(event_name) Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index 12cc5f6e5dd..c60dac6f571 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -100,7 +100,7 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - redis.pipelined do |pipeline| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| pipeline.expire(issue_ids_key, REDIS_EXPIRY_TIME) pipeline.expire(current_index_key, REDIS_EXPIRY_TIME) pipeline.expire(current_project_key, REDIS_EXPIRY_TIME) diff --git a/lib/gitlab/jira/http_client.rb b/lib/gitlab/jira/http_client.rb index 7abfe8e38e8..2b8b01e2023 100644 --- a/lib/gitlab/jira/http_client.rb +++ b/lib/gitlab/jira/http_client.rb @@ -34,6 +34,17 @@ module Gitlab request_params[:headers][:Cookie] = get_cookies if options[:use_cookies] request_params[:base_uri] = uri.to_s request_params.merge!(auth_params) + # Setting defaults here so we can also set `timeout` which prevents setting defaults in the HTTP gem's code + request_params[:open_timeout] = options[:open_timeout] || default_timeout_for(:open_timeout) + request_params[:read_timeout] = options[:read_timeout] || default_timeout_for(:read_timeout) + request_params[:write_timeout] = options[:write_timeout] || default_timeout_for(:write_timeout) + # Global timeout. Needs to be at least as high as the maximum defined in other timeouts + request_params[:timeout] = [ + Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT, + request_params[:open_timeout], + request_params[:read_timeout], + request_params[:write_timeout] + ].max result = Gitlab::HTTP.public_send(http_method, path, **request_params) # rubocop:disable GitlabSecurity/PublicSend @authenticated = result.response.is_a?(Net::HTTPOK) @@ -52,6 +63,10 @@ module Gitlab private + def default_timeout_for(param) + Gitlab::HTTP::DEFAULT_TIMEOUT_OPTIONS[param] + end + def auth_params return {} unless @options[:username] && @options[:password] diff --git a/lib/gitlab/jira/middleware.rb b/lib/gitlab/jira/middleware.rb deleted file mode 100644 index 8a74729da49..00000000000 --- a/lib/gitlab/jira/middleware.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Jira - class Middleware - def self.jira_dvcs_connector?(env) - env['HTTP_USER_AGENT']&.downcase&.start_with?('jira dvcs connector') - end - - def initialize(app) - @app = app - end - - def call(env) - if self.class.jira_dvcs_connector?(env) - env['HTTP_AUTHORIZATION'] = env['HTTP_AUTHORIZATION']&.sub('token', 'Bearer') - end - - @app.call(env) - end - end - end -end diff --git a/lib/gitlab/jira_import/base_importer.rb b/lib/gitlab/jira_import/base_importer.rb index 2b83f0492cb..04ef1a0ef68 100644 --- a/lib/gitlab/jira_import/base_importer.rb +++ b/lib/gitlab/jira_import/base_importer.rb @@ -5,7 +5,7 @@ module Gitlab class BaseImporter attr_reader :project, :client, :formatter, :jira_project_key, :running_import - def initialize(project) + def initialize(project, client = nil) Gitlab::JiraImport.validate_project_settings!(project) @running_import = project.latest_jira_import @@ -14,7 +14,7 @@ module Gitlab raise Projects::ImportService::Error, _('Unable to find Jira project to import data from.') unless @jira_project_key @project = project - @client = project.jira_integration.client + @client = client || project.jira_integration.client @formatter = Gitlab::ImportFormatter.new end diff --git a/lib/gitlab/jira_import/issues_importer.rb b/lib/gitlab/jira_import/issues_importer.rb index 458f7c3f470..54ececc4938 100644 --- a/lib/gitlab/jira_import/issues_importer.rb +++ b/lib/gitlab/jira_import/issues_importer.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :imported_items_cache_key, :start_at, :job_waiter - def initialize(project) + def initialize(project, client = nil) super # get cached start_at value, or zero if not cached yet @start_at = Gitlab::JiraImport.get_issues_next_start_at(project.id) diff --git a/lib/gitlab/job_waiter.rb b/lib/gitlab/job_waiter.rb index e53bfb40654..7b491b3e14d 100644 --- a/lib/gitlab/job_waiter.rb +++ b/lib/gitlab/job_waiter.rb @@ -19,9 +19,6 @@ module Gitlab class JobWaiter KEY_PREFIX = "gitlab:job_waiter" - STARTED_METRIC = :gitlab_job_waiter_started_total - TIMEOUTS_METRIC = :gitlab_job_waiter_timeouts_total - # This TTL needs to be long enough to allow whichever Sidekiq job calls # JobWaiter#wait to reach BLPOP. DEFAULT_TTL = 6.hours.to_i @@ -48,16 +45,15 @@ module Gitlab Gitlab::Redis::SharedState.with { |redis| redis.del(key) } if key?(key) end - attr_reader :key, :finished, :worker_label + attr_reader :key, :finished attr_accessor :jobs_remaining # jobs_remaining - the number of jobs left to wait for # key - The key of this waiter. - def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}", worker_label: nil) + def initialize(jobs_remaining = 0, key = "#{KEY_PREFIX}:#{SecureRandom.uuid}") @key = key @jobs_remaining = jobs_remaining @finished = [] - @worker_label = worker_label end # Waits for all the jobs to be completed. @@ -67,7 +63,6 @@ module Gitlab # long to process, or is never processed. def wait(timeout = 10) deadline = Time.now.utc + timeout - increment_counter(STARTED_METRIC) Gitlab::Redis::SharedState.with do |redis| while jobs_remaining > 0 @@ -81,10 +76,7 @@ module Gitlab list, jid = redis.blpop(key, timeout: seconds_left) # timed out - unless list && jid - increment_counter(TIMEOUTS_METRIC) - break - end + break unless list && jid @finished << jid @jobs_remaining -= 1 @@ -93,20 +85,5 @@ module Gitlab finished end - - private - - def increment_counter(metric) - return unless worker_label - - metrics[metric].increment(worker: worker_label) - end - - def metrics - @metrics ||= { - STARTED_METRIC => Gitlab::Metrics.counter(STARTED_METRIC, 'JobWaiter attempts started'), - TIMEOUTS_METRIC => Gitlab::Metrics.counter(TIMEOUTS_METRIC, 'JobWaiter attempts timed out') - } - end end end diff --git a/lib/gitlab/kubernetes/kubeconfig/template.rb b/lib/gitlab/kubernetes/kubeconfig/template.rb index d40b9ce117e..844472f9c8e 100644 --- a/lib/gitlab/kubernetes/kubeconfig/template.rb +++ b/lib/gitlab/kubernetes/kubeconfig/template.rb @@ -44,7 +44,7 @@ module Gitlab ) end kubeconfig_yaml[:clusters].each do |cluster| - ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.yield_self do |data| + ca_pem = cluster.dig(:cluster, :'certificate-authority-data')&.then do |data| Base64.strict_decode64(data) end diff --git a/lib/gitlab/legacy_http.rb b/lib/gitlab/legacy_http.rb index f38b2819c15..cf6ab80d37f 100644 --- a/lib/gitlab/legacy_http.rb +++ b/lib/gitlab/legacy_http.rb @@ -35,8 +35,8 @@ module Gitlab read_total_timeout = options.fetch(:timeout, Gitlab::HTTP::DEFAULT_READ_TOTAL_TIMEOUT) httparty_perform_request(http_method, path, options_with_timeouts) do |fragment| - start_time ||= Gitlab::Metrics::System.monotonic_time - elapsed = Gitlab::Metrics::System.monotonic_time - start_time + start_time ||= ::Gitlab::Metrics::System.monotonic_time + elapsed = ::Gitlab::Metrics::System.monotonic_time - start_time if elapsed > read_total_timeout raise Gitlab::HTTP::ReadTotalTimeout, "Request timed out after #{elapsed} seconds" diff --git a/lib/gitlab/memory/reporter.rb b/lib/gitlab/memory/reporter.rb index db0fd24983b..8d32745ac34 100644 --- a/lib/gitlab/memory/reporter.rb +++ b/lib/gitlab/memory/reporter.rb @@ -26,13 +26,13 @@ module Gitlab perf_report: report.name )) - start_monotonic_time = Gitlab::Metrics::System.monotonic_time - start_thread_cpu_time = Gitlab::Metrics::System.thread_cpu_time + start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time + start_thread_cpu_time = ::Gitlab::Metrics::System.thread_cpu_time report_file = store_report(report) - cpu_s = Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + cpu_s = ::Gitlab::Metrics::System.thread_cpu_duration(start_thread_cpu_time) + duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time @logger.info( log_labels( diff --git a/lib/gitlab/memory/reports_uploader.rb b/lib/gitlab/memory/reports_uploader.rb index 76c3e0862e2..17230414a6a 100644 --- a/lib/gitlab/memory/reports_uploader.rb +++ b/lib/gitlab/memory/reports_uploader.rb @@ -13,11 +13,11 @@ module Gitlab def upload(path) log_upload_requested(path) - start_monotonic_time = Gitlab::Metrics::System.monotonic_time + start_monotonic_time = ::Gitlab::Metrics::System.monotonic_time File.open(path.to_s) { |file| fog.put_object(gcs_bucket, File.basename(path), file) } - duration_s = Gitlab::Metrics::System.monotonic_time - start_monotonic_time + duration_s = ::Gitlab::Metrics::System.monotonic_time - start_monotonic_time log_upload_success(path, duration_s) rescue StandardError, Errno::ENOENT => error log_exception(error) diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index e18909d8f17..075a897478b 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -5,6 +5,7 @@ module Gitlab class CheckResult SUCCESS_STATUS = :success FAILED_STATUS = :failed + INACTIVE_STATUS = :inactive attr_reader :status, :payload @@ -20,6 +21,10 @@ module Gitlab new(status: FAILED_STATUS, payload: default_payload.merge(**payload)) end + def self.inactive(payload: {}) + new(status: INACTIVE_STATUS, payload: default_payload.merge(**payload)) + end + def self.from_hash(data) new( status: data.fetch(:status).to_sym, diff --git a/lib/gitlab/metrics/exporter/metrics_middleware.rb b/lib/gitlab/metrics/exporter/metrics_middleware.rb index 258b655229e..b80a8c503e8 100644 --- a/lib/gitlab/metrics/exporter/metrics_middleware.rb +++ b/lib/gitlab/metrics/exporter/metrics_middleware.rb @@ -9,10 +9,10 @@ module Gitlab default_labels = { pid: pid } - @requests_total = Gitlab::Metrics.counter( + @requests_total = ::Gitlab::Metrics.counter( :exporter_http_requests_total, 'Total number of HTTP requests', default_labels ) - @request_durations = Gitlab::Metrics.histogram( + @request_durations = ::Gitlab::Metrics.histogram( :exporter_http_request_duration_seconds, 'HTTP request duration histogram (seconds)', default_labels, @@ -21,9 +21,9 @@ module Gitlab end def call(env) - start = Gitlab::Metrics::System.monotonic_time + start = ::Gitlab::Metrics::System.monotonic_time @app.call(env).tap do |response| - duration = Gitlab::Metrics::System.monotonic_time - start + duration = ::Gitlab::Metrics::System.monotonic_time - start labels = { method: env['REQUEST_METHOD'].downcase, diff --git a/lib/gitlab/middleware/go.rb b/lib/gitlab/middleware/go.rb index d2336ec4bb2..5a0612be88e 100644 --- a/lib/gitlab/middleware/go.rb +++ b/lib/gitlab/middleware/go.rb @@ -141,7 +141,7 @@ module Gitlab return empty_result unless has_basic_credentials?(request) login, password = user_name_and_password(request) - auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, ip: request.ip) + auth_result = Gitlab::Auth.find_for_git_client(login, password, project: project, request: request) return empty_result unless auth_result.success? return empty_result unless auth_result.can?(:access_git) diff --git a/lib/gitlab/middleware/path_traversal_check.rb b/lib/gitlab/middleware/path_traversal_check.rb index 79465f3cb30..6fef247b708 100644 --- a/lib/gitlab/middleware/path_traversal_check.rb +++ b/lib/gitlab/middleware/path_traversal_check.rb @@ -5,6 +5,28 @@ module Gitlab class PathTraversalCheck PATH_TRAVERSAL_MESSAGE = 'Potential path traversal attempt detected' + EXCLUDED_EXACT_PATHS = %w[/search].freeze + EXCLUDED_PATH_PREFIXES = %w[/search/].freeze + + EXCLUDED_API_PATHS = %w[/search].freeze + EXCLUDED_PROJECT_API_PATHS = %w[/search].freeze + EXCLUDED_GROUP_API_PATHS = %w[/search].freeze + + API_PREFIX = %r{/api/[^/]+} + API_SUFFIX = %r{(?:\.[^/]+)?} + + EXCLUDED_API_PATHS_REGEX = [ + EXCLUDED_API_PATHS.map do |path| + %r{\A#{API_PREFIX}#{path}#{API_SUFFIX}\z} + end.freeze, + EXCLUDED_PROJECT_API_PATHS.map do |path| + %r{\A#{API_PREFIX}/projects/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z} + end.freeze, + EXCLUDED_GROUP_API_PATHS.map do |path| + %r{\A#{API_PREFIX}/groups/[^/]+(?:/-)?#{path}#{API_SUFFIX}\z} + end.freeze + ].flatten.freeze + def initialize(app) @app = app end @@ -14,7 +36,8 @@ module Gitlab log_params = {} execution_time = measure_execution_time do - check(env, log_params) + request = ::Rack::Request.new(env.dup) + check(request, log_params) unless excluded?(request) end log_params[:duration_ms] = execution_time.round(5) if execution_time @@ -37,17 +60,25 @@ module Gitlab end end - def check(env, log_params) - request = ::Rack::Request.new(env) - fullpath = request.fullpath - decoded_fullpath = CGI.unescape(fullpath) + def check(request, log_params) + decoded_fullpath = CGI.unescape(request.fullpath) ::Gitlab::PathTraversal.check_path_traversal!(decoded_fullpath, skip_decoding: true) - rescue ::Gitlab::PathTraversal::PathTraversalAttackError - log_params[:fullpath] = fullpath + log_params[:method] = request.request_method + log_params[:fullpath] = request.fullpath log_params[:message] = PATH_TRAVERSAL_MESSAGE end + def excluded?(request) + path = request.path + + return true if path.in?(EXCLUDED_EXACT_PATHS) + return true if EXCLUDED_PATH_PREFIXES.any? { |p| path.start_with?(p) } + return true if EXCLUDED_API_PATHS_REGEX.any? { |r| path.match?(r) } + + false + end + def log(payload) Gitlab::AppLogger.warn( payload.merge(class_name: self.class.name) diff --git a/lib/gitlab/omniauth_initializer.rb b/lib/gitlab/omniauth_initializer.rb index 81ad7a7f9e1..0bcd5b1196a 100644 --- a/lib/gitlab/omniauth_initializer.rb +++ b/lib/gitlab/omniauth_initializer.rb @@ -29,6 +29,8 @@ module Gitlab { authorize_params: { gl_auth_type: 'login' } } + when ->(provider_name) { AuthHelper.saml_providers.include?(provider_name.to_sym) } + { attribute_statements: ::Gitlab::Auth::Saml::Config.default_attribute_statements } else {} end @@ -61,7 +63,7 @@ module Gitlab provider_arguments.concat arguments provider_arguments << defaults unless defaults.empty? when Hash, GitlabSettings::Options - hash_arguments = arguments.deep_symbolize_keys.deep_merge(defaults) + hash_arguments = merge_hash_defaults_and_args(defaults, arguments) normalized = normalize_hash_arguments(hash_arguments) # A Hash from the configuration will be passed as is. @@ -80,6 +82,13 @@ module Gitlab provider_arguments end + def merge_hash_defaults_and_args(defaults, arguments) + return arguments.to_hash if defaults.empty? + return defaults.deep_merge(arguments.deep_symbolize_keys) if Feature.enabled?(:invert_omniauth_args_merging) + + arguments.to_hash.deep_symbolize_keys.deep_merge(defaults) + end + def normalize_hash_arguments(args) args.deep_symbolize_keys! diff --git a/lib/gitlab/optimistic_locking.rb b/lib/gitlab/optimistic_locking.rb index 3c8ac55f70b..adc417f287c 100644 --- a/lib/gitlab/optimistic_locking.rb +++ b/lib/gitlab/optimistic_locking.rb @@ -7,7 +7,7 @@ module Gitlab module_function def retry_lock(subject, max_retries = MAX_RETRIES, name:, &block) - start_time = Gitlab::Metrics::System.monotonic_time + start_time = ::Gitlab::Metrics::System.monotonic_time retry_attempts = 0 # prevent scope override, see https://gitlab.com/gitlab-org/gitlab/-/issues/391186 @@ -39,7 +39,7 @@ module Gitlab def log_optimistic_lock_retries(name:, retry_attempts:, start_time:) return unless retry_attempts > 0 - elapsed_time = Gitlab::Metrics::System.monotonic_time - start_time + elapsed_time = ::Gitlab::Metrics::System.monotonic_time - start_time retry_lock_logger.info( message: "Optimistic Lock released with retries", diff --git a/lib/gitlab/pages/deployment_update.rb b/lib/gitlab/pages/deployment_update.rb index 6845f5d88ec..bf6ac3a056d 100644 --- a/lib/gitlab/pages/deployment_update.rb +++ b/lib/gitlab/pages/deployment_update.rb @@ -89,14 +89,10 @@ module Gitlab project.actual_limits.pages_file_entries end + # If a newer pipeline already build a PagesDeployment def validate_outdated_sha return if latest? - - # use pipeline_id in case the build is retried - last_deployed_pipeline_id = project.pages_metadatum&.pages_deployment&.ci_build&.pipeline_id - - return unless last_deployed_pipeline_id - return if last_deployed_pipeline_id <= build.pipeline_id + return if latest_pipeline_id <= build.pipeline_id errors.add(:base, 'build SHA is outdated for this ref') end @@ -111,6 +107,13 @@ module Gitlab def sha build.sha end + + def latest_pipeline_id + project + .active_pages_deployments + .with_path_prefix(build.pages&.dig(:path_prefix)) + .latest_pipeline_id + end end end end diff --git a/lib/gitlab/pagination/cursor_based_keyset.rb b/lib/gitlab/pagination/cursor_based_keyset.rb index 81dcc54ff35..9e8c0c530a9 100644 --- a/lib/gitlab/pagination/cursor_based_keyset.rb +++ b/lib/gitlab/pagination/cursor_based_keyset.rb @@ -34,8 +34,10 @@ module Gitlab order_satisfied?(relation, cursor_based_request_context) end - def self.enforced_for_type?(relation) - ENFORCED_TYPES.include?(relation.klass) + def self.enforced_for_type?(request_scope, relation) + enforced = ENFORCED_TYPES + enforced += [::Ci::Build] if ::Feature.enabled?(:enforce_ci_builds_pagination_limit, request_scope, type: :ops) + enforced.include?(relation.klass) end def self.order_satisfied?(relation, cursor_based_request_context) diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb index c9eae2f899f..8f1fbf53161 100644 --- a/lib/gitlab/patch/sidekiq_cron_poller.rb +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -7,7 +7,7 @@ require 'sidekiq/version' require 'sidekiq/cron/version' -if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.7') +if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.12') raise 'New version of sidekiq detected, please remove or update this patch' end diff --git a/lib/gitlab/patch/sidekiq_scheduled_enq.rb b/lib/gitlab/patch/sidekiq_scheduled_enq.rb index de0e8465f97..b5a40c19923 100644 --- a/lib/gitlab/patch/sidekiq_scheduled_enq.rb +++ b/lib/gitlab/patch/sidekiq_scheduled_enq.rb @@ -15,10 +15,8 @@ module Gitlab # this portion swaps out Sidekiq.redis for Gitlab::Redis::Queues Gitlab::Redis::Queues.with do |conn| # rubocop:disable Cop/RedisQueueUsage sorted_sets.each do |sorted_set| - # adds namespace if `super` polls with a non-namespaced Sidekiq.redis - if Gitlab::Utils.to_boolean(ENV['SIDEKIQ_ENQUEUE_NON_NAMESPACED']) - sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage - end + # adds namespace since `super` polls with a non-namespaced Sidekiq.redis + sorted_set = "#{Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE}:#{sorted_set}" # rubocop:disable Cop/RedisQueueUsage while !@done && (job = zpopbyscore(conn, keys: [sorted_set], argv: [Time.now.to_f.to_s])) # rubocop:disable Gitlab/ModuleWithInstanceVariables, Lint/AssignmentInCondition Sidekiq::Client.push(Sidekiq.load_json(job)) # rubocop:disable Cop/SidekiqApiUsage @@ -28,7 +26,6 @@ module Gitlab end end - # calls original enqueue_jobs which may or may not be namespaced depending on SIDEKIQ_ENQUEUE_NON_NAMESPACED super end end diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index c9ed4720e83..5f2084ce011 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -60,14 +60,14 @@ module Gitlab ProjectTemplate.new('dotnetcore', '.NET Core', _('A .NET Core console application template, customizable for any .NET Core project'), 'https://gitlab.com/gitlab-org/project-templates/dotnetcore', 'illustrations/third-party-logos/dotnet.svg'), ProjectTemplate.new('android', 'Android', _('A ready-to-go template for use with Android apps'), 'https://gitlab.com/gitlab-org/project-templates/android', 'illustrations/logos/android.svg'), ProjectTemplate.new('gomicro', 'Go Micro', _('Go Micro is a framework for micro service development'), 'https://gitlab.com/gitlab-org/project-templates/go-micro', 'illustrations/logos/gomicro.svg'), - ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/gitlab-org/project-templates/bridgetown'), + ProjectTemplate.new('bridgetown', 'Pages/Bridgetown', _('Everything you need to create a GitLab Pages site using Bridgetown'), 'https://gitlab.com/pages/bridgetown'), ProjectTemplate.new('gatsby', 'Pages/Gatsby', _('Everything you need to create a GitLab Pages site using Gatsby'), 'https://gitlab.com/pages/gatsby', 'illustrations/third-party-logos/gatsby.svg'), ProjectTemplate.new('hugo', 'Pages/Hugo', _('Everything you need to create a GitLab Pages site using Hugo'), 'https://gitlab.com/pages/hugo', 'illustrations/logos/hugo.svg'), ProjectTemplate.new('pelican', 'Pages/Pelican', _('Everything you need to create a GitLab Pages site using Pelican'), 'https://gitlab.com/pages/pelican', 'illustrations/third-party-logos/pelican.svg'), ProjectTemplate.new('jekyll', 'Pages/Jekyll', _('Everything you need to create a GitLab Pages site using Jekyll'), 'https://gitlab.com/pages/jekyll', 'illustrations/logos/jekyll.svg'), ProjectTemplate.new('plainhtml', 'Pages/Plain HTML', _('Everything you need to create a GitLab Pages site using plain HTML'), 'https://gitlab.com/pages/plain-html'), ProjectTemplate.new('hexo', 'Pages/Hexo', _('Everything you need to create a GitLab Pages site using Hexo'), 'https://gitlab.com/pages/hexo', 'illustrations/logos/hexo.svg'), - ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/gitlab-org/project-templates/middleman', 'illustrations/logos/middleman.svg'), + ProjectTemplate.new('middleman', 'Pages/Middleman', _('Everything you need to create a GitLab Pages site using Middleman'), 'https://gitlab.com/pages/middleman', 'illustrations/logos/middleman.svg'), ProjectTemplate.new('gitpod_spring_petclinic', 'Gitpod/Spring Petclinic', _('A Gitpod configured Webapplication in Spring and Java'), 'https://gitlab.com/gitlab-org/project-templates/gitpod-spring-petclinic', 'illustrations/logos/gitpod.svg'), ProjectTemplate.new('nfhugo', 'Netlify/Hugo', _('A Hugo site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfhugo', 'illustrations/logos/netlify.svg'), ProjectTemplate.new('nfjekyll', 'Netlify/Jekyll', _('A Jekyll site that uses Netlify for CI/CD instead of GitLab, but still with all the other great GitLab features'), 'https://gitlab.com/pages/nfjekyll', 'illustrations/logos/netlify.svg'), @@ -81,7 +81,8 @@ module Gitlab ProjectTemplate.new('cluster_management', 'GitLab Cluster Management', _('An example project for managing Kubernetes clusters integrated with GitLab'), 'https://gitlab.com/gitlab-org/project-templates/cluster-management'), ProjectTemplate.new('kotlin_native_linux', 'Kotlin Native Linux', _('A basic template for developing Linux programs using Kotlin Native'), 'https://gitlab.com/gitlab-org/project-templates/kotlin-native-linux'), ProjectTemplate.new('typo3_distribution', 'TYPO3 Distribution', _('A template for starting a new TYPO3 project'), 'https://gitlab.com/gitlab-org/project-templates/typo3-distribution', 'illustrations/logos/typo3.svg'), - ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg') + ProjectTemplate.new('laravel', 'Laravel Framework', _('A basic folder structure of a Laravel application, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/laravel', 'illustrations/logos/laravel.svg'), + ProjectTemplate.new('astro_tailwind', 'Astro Tailwind', _('A basic folder structure of Astro Starter Kit, to help you get started.'), 'https://gitlab.com/gitlab-org/project-templates/astro-tailwind') ] end # rubocop:enable Metrics/AbcSize diff --git a/lib/gitlab/push_options.rb b/lib/gitlab/push_options.rb index 4471d21b9ac..e817f2130f4 100644 --- a/lib/gitlab/push_options.rb +++ b/lib/gitlab/push_options.rb @@ -14,6 +14,7 @@ module Gitlab :milestone, :remove_source_branch, :target, + :target_project, :title, :unassign, :unlabel diff --git a/lib/gitlab/quick_actions/merge_request_actions.rb b/lib/gitlab/quick_actions/merge_request_actions.rb index 9798b0eca2c..72bec159226 100644 --- a/lib/gitlab/quick_actions/merge_request_actions.rb +++ b/lib/gitlab/quick_actions/merge_request_actions.rb @@ -172,6 +172,25 @@ module Gitlab end end + desc { _('Request changes') } + explanation { _('Request changes to the current merge request.') } + types MergeRequest + condition do + Feature.enabled?(:mr_request_changes, current_user) && + quick_action_target.persisted? && + quick_action_target.find_reviewer(current_user) + end + command :request_changes do + result = ::MergeRequests::UpdateReviewerStateService.new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, "requested_changes") + + @execution_message[:request_changes] = if result[:status] == :success + _('Changes requested to the current merge request.') + else + result[:message] + end + end + desc { _('Approve a merge request') } explanation { _('Approve the current merge request.') } types MergeRequest @@ -197,6 +216,10 @@ module Gitlab next unless success + ::MergeRequests::UpdateReviewerStateService + .new(project: quick_action_target.project, current_user: current_user) + .execute(quick_action_target, "unreviewed") + @execution_message[:unapprove] = _('Unapproved the current merge request.') end diff --git a/lib/gitlab/redis.rb b/lib/gitlab/redis.rb index 89ec996488f..9f7599d2500 100644 --- a/lib/gitlab/redis.rb +++ b/lib/gitlab/redis.rb @@ -14,7 +14,6 @@ module Gitlab Gitlab::Redis::FeatureFlag, Gitlab::Redis::Queues, Gitlab::Redis::QueuesMetadata, - Gitlab::Redis::Pubsub, Gitlab::Redis::RateLimiting, Gitlab::Redis::RepositoryCache, Gitlab::Redis::Sessions, diff --git a/lib/gitlab/redis/cluster_util.rb b/lib/gitlab/redis/cluster_util.rb index 5f1f39b5237..9e307940de3 100644 --- a/lib/gitlab/redis/cluster_util.rb +++ b/lib/gitlab/redis/cluster_util.rb @@ -26,6 +26,15 @@ module Gitlab end expired_count end + + # Redis cluster alternative to mget + def batch_get(keys, redis) + keys.each_slice(1000).flat_map do |subset| + Gitlab::Redis::CrossSlot::Pipeline.new(redis).pipelined do |pipeline| + subset.map { |key| pipeline.get(key) } + end + end + end end end end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index bbe5a8add4b..6acbf83df24 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -63,8 +63,12 @@ module Gitlab hlen hmget hscan_each + llen + lrange mapped_hmget mget + pfcount + pttl scan scan_each scard @@ -72,20 +76,32 @@ module Gitlab smembers sscan sscan_each + strlen ttl + type + zcard + zcount + zrange + zrangebyscore + zrevrange zscan_each + zscore ].freeze WRITE_COMMANDS = %i[ + decr del eval expire flushdb hdel + hincrby hset incr incrby mapped_hmset + pfadd + pfmerge publish rpush sadd @@ -93,8 +109,15 @@ module Gitlab set setex setnx + spop srem + srem? unlink + zadd + zpopmin + zrem + zremrangebyrank + zremrangebyscore memory ].freeze @@ -254,11 +277,27 @@ module Gitlab # # Let's define it explicitly instead of propagating it to method_missing def close - if use_primary_and_secondary_stores? - [primary_store, secondary_store].map(&:close).first + if same_redis_store? + # if same_redis_store?, `use_primary_store_as_default?` returns false + # but we should avoid a feature-flag check in `.close` to avoid checking out + # an ActiveRecord connection during clean up. + secondary_store.close else - default_store.close + [primary_store, secondary_store].map(&:close).first + end + end + + # blpop blocks until an element to be popped exist in the list or after a timeout. + def blpop(*args) + result = default_store.blpop(*args) + if !!result && use_primary_and_secondary_stores? + # special case to accommodate Gitlab::JobWaiter as blpop is only used in JobWaiter + # 1s should be sufficient wait time to account for delays between 1st and 2nd lpush + # https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/2520#note_1630893702 + non_default_store.blpop(args.first, timeout: 1) end + + result end private @@ -380,7 +419,7 @@ module Gitlab end def redis_store?(store) - store.is_a?(::Redis) || store.is_a?(::Redis::Namespace) + store.is_a?(::Redis) end def validate_stores! diff --git a/lib/gitlab/redis/pubsub.rb b/lib/gitlab/redis/pubsub.rb deleted file mode 100644 index b5022f467a2..00000000000 --- a/lib/gitlab/redis/pubsub.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Redis - class Pubsub < ::Gitlab::Redis::Wrapper - class << self - def config_fallback - SharedState - end - end - end - end -end diff --git a/lib/gitlab/redis/shared_state.rb b/lib/gitlab/redis/shared_state.rb index fb3a143121b..d12d3e8c6aa 100644 --- a/lib/gitlab/redis/shared_state.rb +++ b/lib/gitlab/redis/shared_state.rb @@ -3,6 +3,12 @@ module Gitlab module Redis class SharedState < ::Gitlab::Redis::Wrapper + def self.redis + primary_store = ::Redis.new(ClusterSharedState.params) + secondary_store = ::Redis.new(params) + + MultiStore.new(primary_store, secondary_store, store_name) + end end end end diff --git a/lib/gitlab/redis/wrapper.rb b/lib/gitlab/redis/wrapper.rb index 2bcf4769b5a..d5470bc0016 100644 --- a/lib/gitlab/redis/wrapper.rb +++ b/lib/gitlab/redis/wrapper.rb @@ -19,7 +19,7 @@ module Gitlab InvalidPathError = Class.new(StandardError) class << self - delegate :params, :url, :store, to: :new + delegate :params, :url, :store, :encrypted_secrets, to: :new def with pool.with { |redis| yield redis } @@ -110,6 +110,14 @@ module Gitlab raw_config_hash[:sentinels] end + def secret_file + if raw_config_hash[:secret_file].blank? + File.join(Settings.encrypted_settings['path'], 'redis.yaml.enc') + else + Settings.absolute(raw_config_hash[:secret_file]) + end + end + def sentinels? sentinels && !sentinels.empty? end @@ -118,22 +126,44 @@ module Gitlab ::Redis::Store::Factory.create(redis_store_options.merge(extras)) end + def encrypted_secrets + # In rake tasks, we have to populate the encrypted_secrets even if the + # file does not exist, as it is the job of one of those tasks to create + # the file. In other cases, like when being loaded as part of spinning + # up test environment via `scripts/setup-test-env`, we should gate on + # the presence of the specified secret file so that + # `Settings.encrypted`, which might not be loadable does not gets + # called. + Settings.encrypted(secret_file) if File.exist?(secret_file) || ::Gitlab::Runtime.rake? + end + private def redis_store_options config = raw_config_hash config[:instrumentation_class] ||= self.class.instrumentation_class - result = if config[:cluster].present? - config[:db] = 0 # Redis Cluster only supports db 0 - config + decrypted_config = parse_encrypted_config(config) + + result = if decrypted_config[:cluster].present? + decrypted_config[:db] = 0 # Redis Cluster only supports db 0 + decrypted_config else - parse_redis_url(config) + parse_redis_url(decrypted_config) end parse_client_tls_options(result) end + def parse_encrypted_config(encrypted_config) + encrypted_config.delete(:secret_file) + + decrypted_secrets = encrypted_secrets&.config + encrypted_config.merge!(decrypted_secrets) if decrypted_secrets + + encrypted_config + end + def parse_redis_url(config) redis_url = config.delete(:url) redis_uri = URI.parse(redis_url) diff --git a/lib/gitlab/regex.rb b/lib/gitlab/regex.rb index 2fd9dc9fa09..6ac37986d5c 100644 --- a/lib/gitlab/regex.rb +++ b/lib/gitlab/regex.rb @@ -5,6 +5,7 @@ module Gitlab extend self extend MergeRequests extend Packages + extend Packages::Protection::Rules def project_name_regex # The character range \p{Alnum} overlaps with \u{00A9}-\u{1f9ff} diff --git a/lib/gitlab/regex/packages.rb b/lib/gitlab/regex/packages.rb index 6b178933a25..a0038d39318 100644 --- a/lib/gitlab/regex/packages.rb +++ b/lib/gitlab/regex/packages.rb @@ -3,6 +3,8 @@ module Gitlab module Regex module Packages + include ::Gitlab::Utils::StrongMemoize + CONAN_RECIPE_FILES = %w[conanfile.py conanmanifest.txt conan_sources.tgz conan_export.tgz].freeze CONAN_PACKAGE_FILES = %w[conaninfo.txt conanmanifest.txt conan_package.tgz].freeze @@ -74,8 +76,10 @@ module Gitlab maven_app_name_regex end - def npm_package_name_regex - @npm_package_name_regex ||= %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9]+\z}o + def npm_package_name_regex(other_accepted_chars = nil) + strong_memoize_with(:npm_package_name_regex, other_accepted_chars) do + %r{\A(?:@(#{Gitlab::PathRegex::NAMESPACE_FORMAT_REGEX})/)?[-+\.\_a-zA-Z0-9#{other_accepted_chars}]+\z} + end end def npm_package_name_regex_message diff --git a/lib/gitlab/regex/packages/protection/rules.rb b/lib/gitlab/regex/packages/protection/rules.rb new file mode 100644 index 00000000000..383f26fe92d --- /dev/null +++ b/lib/gitlab/regex/packages/protection/rules.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Regex + module Packages + module Protection + module Rules + def protection_rules_npm_package_name_pattern_regex + @protection_rules_npm_package_name_pattern_regex ||= npm_package_name_regex('*') + end + end + end + end + end +end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 3a389d3363f..d5e80053772 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -6,7 +6,8 @@ module Gitlab module RequestForgeryProtection - class Controller < BaseActionController + # rubocop:disable Rails/ApplicationController + class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true def initialize @@ -39,5 +40,6 @@ module Gitlab rescue ActionController::InvalidAuthenticityToken false end + # rubocop:enable Rails/ApplicationController end end diff --git a/lib/gitlab/rugged_instrumentation.rb b/lib/gitlab/rugged_instrumentation.rb deleted file mode 100644 index 36a3a491de6..00000000000 --- a/lib/gitlab/rugged_instrumentation.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module RuggedInstrumentation - def self.query_time - query_time = SafeRequestStore[:rugged_query_time] || 0 - query_time.round(Gitlab::InstrumentationHelper::DURATION_PRECISION) - end - - def self.add_query_time(duration) - SafeRequestStore[:rugged_query_time] ||= 0 - SafeRequestStore[:rugged_query_time] += duration - end - - def self.query_time_ms - (self.query_time * 1000).round(2) - end - - def self.query_count - SafeRequestStore[:rugged_call_count] ||= 0 - end - - def self.increment_query_count - SafeRequestStore[:rugged_call_count] ||= 0 - SafeRequestStore[:rugged_call_count] += 1 - end - - def self.active? - SafeRequestStore.active? - end - - def self.add_call_details(details) - return unless Gitlab::PerformanceBar.enabled_for_request? - - Gitlab::SafeRequestStore[:rugged_call_details] ||= [] - Gitlab::SafeRequestStore[:rugged_call_details] << details - end - - def self.list_call_details - return [] unless Gitlab::PerformanceBar.enabled_for_request? - - Gitlab::SafeRequestStore[:rugged_call_details] || [] - end - end -end diff --git a/lib/gitlab/search_results.rb b/lib/gitlab/search_results.rb index d06f414bd9a..fada3b84401 100644 --- a/lib/gitlab/search_results.rb +++ b/lib/gitlab/search_results.rb @@ -191,9 +191,7 @@ module Gitlab unless default_project_filter project_ids = project_ids_relation - if Feature.enabled?(:search_issues_hide_archived_projects, current_user) && !filters[:include_archived] - project_ids = project_ids.non_archived - end + project_ids = project_ids.non_archived unless filters[:include_archived] issues = issues.in_projects(project_ids) .allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/420046') @@ -218,9 +216,7 @@ module Gitlab unless default_project_filter project_ids = project_ids_relation - if Feature.enabled?(:search_merge_requests_hide_archived_projects, current_user) && !filters[:include_archived] - project_ids = project_ids.non_archived - end + project_ids = project_ids.non_archived unless filters[:include_archived] merge_requests = merge_requests.of_projects(project_ids) end diff --git a/lib/gitlab/seeders/ci/catalog/resource_seeder.rb b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb new file mode 100644 index 00000000000..2971dabe044 --- /dev/null +++ b/lib/gitlab/seeders/ci/catalog/resource_seeder.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +module Gitlab + module Seeders + module Ci + module Catalog + class ResourceSeeder + # This is currently disabled until it gets fixed: https://gitlab.com/gitlab-org/gitlab/-/issues/429649 + # Initializes the class + # + # @param [String] Path of the group to find + # @param [Integer] Number of resources to create + def initialize(group_path:, seed_count:) + @group = Group.find_by_full_path(group_path) + @seed_count = seed_count + @current_user = @group&.first_owner + end + + def seed + if @group.nil? + warn 'ERROR: Group was not found.' + return + end + + @seed_count.times do |i| + create_ci_catalog_resource(i) + end + end + + private + + def create_project(name, index) + project = ::Projects::CreateService.new( + @current_user, + description: "This is Catalog resource ##{index}", + name: name, + namespace_id: @group.id, + path: name, + visibility_level: @group.visibility_level + ).execute + + if project.saved? + project + else + warn project.errors.full_messages.to_sentence + nil + end + end + + def create_template_yml(project) + template_content = <<~YAML + spec: + inputs: + stage: + default: test + --- + component-job: + script: echo job 1 + stage: $[[ inputs.stage ]] + YAML + + project.repository.create_file( + @current_user, + 'template.yml', + template_content, + message: 'Add template.yml', + branch_name: project.default_branch_or_main + ) + end + + def create_readme(project, index) + project.repository.create_file( + @current_user, + '/README.md', + "## Component stuff #{index}", + message: 'Add README.md', + branch_name: project.default_branch_or_main + ) + end + + def create_ci_catalog(project) + result = ::Ci::Catalog::Resources::CreateService.new(project, @current_user).execute + if result.success? + result.payload + else + warn "Project '#{project.name}' could not be converted to a Catalog resource" + nil + end + end + + def create_ci_catalog_resource(index) + name = "ci_seed_resource_#{index}" + + if Project.find_by_name(name).present? + warn "Project '#{name}' already exists!" + return + end + + project = create_project(name, index) + + return unless project + + create_readme(project, index) + create_template_yml(project) + + return unless create_ci_catalog(project) + + warn "Project '#{name}' was saved successfully!" + end + end + end + end + end +end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index a1363e7b6b2..10a69acc037 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -21,6 +21,7 @@ module Gitlab include Gitlab::Utils::StrongMemoize DEFAULT_DUPLICATE_KEY_TTL = 6.hours + SHORT_DUPLICATE_KEY_TTL = 10.minutes DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none @@ -134,7 +135,7 @@ module Gitlab jid != existing_jid end - def set_deduplicated_flag!(expiry = duplicate_key_ttl) + def set_deduplicated_flag! return unless reschedulable? with_redis { |redis| redis.eval(DEDUPLICATED_SCRIPT, keys: [cookie_key]) } @@ -173,7 +174,7 @@ module Gitlab end def duplicate_key_ttl - options[:ttl] || DEFAULT_DUPLICATE_KEY_TTL + options[:ttl] || default_duplicate_key_ttl end private @@ -182,6 +183,12 @@ module Gitlab attr_reader :queue_name, :job attr_writer :existing_jid + def default_duplicate_key_ttl + return SHORT_DUPLICATE_KEY_TTL if Feature.enabled?(:reduce_duplicate_job_key_ttl) + + DEFAULT_DUPLICATE_KEY_TTL + end + def worker_klass @worker_klass ||= worker_class_name.to_s.safe_constantize end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb index b065190f656..e7ce837de29 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/strategies/deduplicates_when_scheduling.rb @@ -20,7 +20,7 @@ module Gitlab if duplicate_job.idempotent? duplicate_job.update_latest_wal_location! - duplicate_job.set_deduplicated_flag!(expiry) + duplicate_job.set_deduplicated_flag! Gitlab::SidekiqLogging::DeduplicationLogger.instance.deduplicated_log( job, strategy_name, duplicate_job.options) diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index a8b3683e09f..37a9ed37891 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -25,11 +25,6 @@ module Gitlab def metrics metrics = { - sidekiq_jobs_cpu_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_db_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_jobs_gitaly_seconds: ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS), - sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS), - sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS), sidekiq_jobs_retried_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'), sidekiq_jobs_interrupted_total: ::Gitlab::Metrics.counter(:sidekiq_jobs_interrupted_total, 'Sidekiq jobs interrupted'), sidekiq_redis_requests_total: ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'), @@ -43,9 +38,24 @@ module Gitlab metrics[:sidekiq_jobs_completion_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_completion_seconds, 'Seconds to complete Sidekiq job', {}, SIDEKIQ_JOB_DURATION_BUCKETS) metrics[:sidekiq_jobs_queue_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_QUEUE_DURATION_BUCKETS) metrics[:sidekiq_jobs_failed_total] = ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed') + + # resource usage + metrics[:sidekiq_jobs_cpu_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_cpu_seconds, 'Seconds this Sidekiq job spent on the CPU', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_jobs_db_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_db_seconds, 'Seconds of database time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_jobs_gitaly_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS) + metrics[:sidekiq_redis_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS) + metrics[:sidekiq_elasticsearch_requests_duration_seconds] = ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS) else - # The sum metric is still used in GitLab.com for dashboards + # These metrics are used in GitLab.com dashboards metrics[:sidekiq_jobs_completion_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_seconds_sum, 'Total of seconds to complete Sidekiq job') + metrics[:sidekiq_jobs_completion_count] = ::Gitlab::Metrics.counter(:sidekiq_jobs_completion_count, 'Number of Sidekiq jobs completed') + + # resource usage sums + metrics[:sidekiq_jobs_cpu_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_cpu_seconds_sum, 'Total seconds this Sidekiq job spent on the CPU') + metrics[:sidekiq_jobs_db_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_db_seconds_sum, 'Total seconds of database time to run Sidekiq job') + metrics[:sidekiq_jobs_gitaly_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_jobs_gitaly_seconds_sum, 'Total seconds Gitaly time to run Sidekiq job') + metrics[:sidekiq_redis_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_redis_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to a Redis server') + metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum] = ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_duration_seconds_sum, 'Total duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server') end metrics @@ -89,8 +99,9 @@ module Gitlab # in metrics and can use them in the `ThreadsSampler` for setting a label Thread.current.name ||= Gitlab::Metrics::Samplers::ThreadsSampler::SIDEKIQ_WORKER_THREAD_NAME - labels = create_labels(worker.class, queue, job) - instrument(job, labels) do + @job = job + @labels = create_labels(worker.class, queue, job) + instrument do yield end end @@ -99,8 +110,8 @@ module Gitlab attr_reader :metrics - def instrument(job, labels) - queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) + def instrument + @queue_duration = ::Gitlab::InstrumentationHelper.queue_duration_for_job(job) @metrics[:sidekiq_jobs_queue_duration_seconds]&.observe(labels, queue_duration) if queue_duration @@ -114,43 +125,33 @@ module Gitlab @metrics[:sidekiq_jobs_interrupted_total].increment(labels, 1) end - job_succeeded = false + @job_succeeded = false monotonic_time_start = Gitlab::Metrics::System.monotonic_time job_thread_cputime_start = get_thread_cputime begin transaction = Gitlab::Metrics::BackgroundTransaction.new transaction.run { yield } - job_succeeded = true + @job_succeeded = true ensure monotonic_time_end = Gitlab::Metrics::System.monotonic_time job_thread_cputime_end = get_thread_cputime - monotonic_time = monotonic_time_end - monotonic_time_start - job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start + @monotonic_time = monotonic_time_end - monotonic_time_start + @job_thread_cputime = job_thread_cputime_end - job_thread_cputime_start - # sidekiq_running_jobs, sidekiq_jobs_failed_total should not include the job_status label @metrics[:sidekiq_running_jobs].increment(labels, -1) - if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) - @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded - else - # we don't need job_status label here - @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time) - end + @instrumentation = job[:instrumentation] || {} + + record_resource_usage_counters # job_status: done, fail match the job_status attribute in structured logging labels[:job_status] = job_succeeded ? "done" : "fail" - instrumentation = job[:instrumentation] || {} - @metrics[:sidekiq_jobs_cpu_seconds].observe(labels, job_thread_cputime) - @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time) + record_histograms - @metrics[:sidekiq_jobs_db_seconds].observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) - @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(instrumentation)) @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(instrumentation)) - @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(instrumentation)) @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(instrumentation)) - @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(instrumentation)) @metrics[:sidekiq_mem_total_bytes].set(labels, get_thread_memory_total_allocations(instrumentation)) with_load_balancing_settings(job) do |settings| @@ -162,15 +163,50 @@ module Gitlab @metrics[:sidekiq_load_balancing_count].increment(labels.merge(load_balancing_labels), 1) end - sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) - Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded - Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) - Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + @sli_labels = labels.slice(*SIDEKIQ_SLI_LABELS) + record_execution_sli + record_queueing_sli end end private + attr_reader :labels, :job, :queue_duration, :job_succeeded, :monotonic_time, :job_thread_cputime, :instrumentation, :sli_labels + + def record_resource_usage_counters + if Feature.enabled?(:emit_sidekiq_histogram_metrics, type: :ops) + @metrics[:sidekiq_jobs_failed_total].increment(labels, 1) unless job_succeeded + else + @metrics[:sidekiq_jobs_completion_seconds_sum].increment(labels, monotonic_time) + @metrics[:sidekiq_jobs_completion_count].increment(labels, 1) + @metrics[:sidekiq_jobs_cpu_seconds_sum].increment(labels, job_thread_cputime) + @metrics[:sidekiq_jobs_db_seconds_sum].increment(labels, ActiveRecord::LogSubscriber.runtime / 1000) + @metrics[:sidekiq_jobs_gitaly_seconds_sum].increment(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds_sum].increment(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds_sum].increment(labels, get_elasticsearch_time(instrumentation)) + end + end + + def record_histograms + @metrics[:sidekiq_jobs_cpu_seconds]&.observe(labels, job_thread_cputime) + + @metrics[:sidekiq_jobs_completion_seconds]&.observe(labels, monotonic_time) + + @metrics[:sidekiq_jobs_db_seconds]&.observe(labels, ActiveRecord::LogSubscriber.runtime / 1000) + @metrics[:sidekiq_jobs_gitaly_seconds]&.observe(labels, get_gitaly_time(instrumentation)) + @metrics[:sidekiq_redis_requests_duration_seconds]&.observe(labels, get_redis_time(instrumentation)) + @metrics[:sidekiq_elasticsearch_requests_duration_seconds]&.observe(labels, get_elasticsearch_time(instrumentation)) + end + + def record_queueing_sli + Gitlab::Metrics::SidekiqSlis.record_queueing_apdex(sli_labels, queue_duration) if queue_duration + end + + def record_execution_sli + Gitlab::Metrics::SidekiqSlis.record_execution_apdex(sli_labels, monotonic_time) if job_succeeded + Gitlab::Metrics::SidekiqSlis.record_execution_error(sli_labels, !job_succeeded) + end + def with_load_balancing_settings(job) keys = %w[load_balancing_strategy worker_data_consistency] return unless keys.all? { |k| job.key?(k) } diff --git a/lib/gitlab/sidekiq_middleware/skip_jobs.rb b/lib/gitlab/sidekiq_middleware/skip_jobs.rb index 34ad843e8ee..56b150116a3 100644 --- a/lib/gitlab/sidekiq_middleware/skip_jobs.rb +++ b/lib/gitlab/sidekiq_middleware/skip_jobs.rb @@ -80,13 +80,20 @@ module Gitlab end health_check_attrs = worker_class.database_health_check_attrs - job_base_model = Gitlab::Database.schemas_to_base_models[health_check_attrs[:gitlab_schema]].first + + tables, schema = health_check_attrs.values_at(:tables, :gitlab_schema) + + if health_check_attrs[:block].respond_to?(:call) + schema, tables = health_check_attrs[:block].call(job['args'], schema, tables) + end + + job_base_model = Gitlab::Database.schemas_to_base_models[schema].first health_context = Gitlab::Database::HealthStatus::Context.new( DatabaseHealthStatusChecker.new(job['jid'], worker_class.name), job_base_model.connection, - health_check_attrs[:tables], - health_check_attrs[:gitlab_schema] + tables, + schema ) Gitlab::Database::HealthStatus.evaluate(health_context).any?(&:stop?) diff --git a/lib/gitlab/sidekiq_status.rb b/lib/gitlab/sidekiq_status.rb index 778d278146d..ae4aca7ff92 100644 --- a/lib/gitlab/sidekiq_status.rb +++ b/lib/gitlab/sidekiq_status.rb @@ -94,8 +94,17 @@ module Gitlab keys = job_ids.map { |jid| key_for(jid) } - with_redis { |redis| redis.mget(*keys) } - .map { |result| !result.nil? } + status = with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + if Gitlab::Redis::ClusterUtil.cluster?(redis) + Gitlab::Redis::ClusterUtil.batch_get(keys, redis) + else + redis.mget(*keys) + end + end + end + + status.map { |result| !result.nil? } end # Returns the JIDs that are completed diff --git a/lib/gitlab/tracking.rb b/lib/gitlab/tracking.rb index f127e14243c..3bbcd59f45e 100644 --- a/lib/gitlab/tracking.rb +++ b/lib/gitlab/tracking.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module Tracking class << self diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index 1b7dcaa5cf4..a9b8dc313d0 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -40,6 +40,8 @@ module Gitlab note_url(object, **options) when Release instance.release_url(object, **options) + when Organizations::Organization + instance.organization_url(object, **options) when Project instance.project_url(object, **options) when Snippet diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index 7252283d1b9..941c2f793c4 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -3,7 +3,7 @@ module Gitlab module Usage class MetricDefinition - METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') + METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema', '**', '*.json') AVAILABLE_STATUSES = %w[active broken].to_set.freeze VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze @@ -52,7 +52,7 @@ module Gitlab end def validate! - self.class.schemer.validate(attributes.deep_stringify_keys).each do |error| + errors.each do |error| error_message = <<~ERROR_MSG Error type: #{error['type']} Data: #{error['data']} @@ -104,8 +104,10 @@ module Gitlab definitions[key_path]&.to_context end - def schemer - @schemer ||= ::JSONSchemer.schema(Pathname.new(METRIC_SCHEMA_PATH)) + def schemers + @schemers ||= Dir[METRIC_SCHEMA_PATH].map do |path| + ::JSONSchemer.schema(Pathname.new(path)) + end end def dump_metrics_yaml @@ -145,6 +147,19 @@ module Gitlab private + def errors + result = [] + + self.class.schemers.each do |schemer| + # schemer.validate returns an Enumerator object + schemer.validate(attributes.deep_stringify_keys).each do |error| + result << error + end + end + + result + end + def method_missing(method, *args) attributes[method] || super end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index b2027791e9d..5f819f060e4 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -174,7 +174,6 @@ module Gitlab prometheus_enabled: alt_usage_data(fallback: nil) { Gitlab::Prometheus::Internal.prometheus_enabled? }, prometheus_metrics_enabled: alt_usage_data(fallback: nil) { Gitlab::Metrics.prometheus_metrics_enabled? }, reply_by_email_enabled: alt_usage_data(fallback: nil) { Gitlab::Email::IncomingEmail.enabled? }, - web_ide_clientside_preview_enabled: alt_usage_data(fallback: nil) { false }, signup_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.allow_signup? }, grafana_link_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.grafana_enabled? }, gitpod_enabled: alt_usage_data(fallback: nil) { Gitlab::CurrentSettings.gitpod_enabled? } diff --git a/lib/gitlab/usage_data_counters/hll_redis_counter.rb b/lib/gitlab/usage_data_counters/hll_redis_counter.rb index f9dc8bd8a3c..185b49d4a68 100644 --- a/lib/gitlab/usage_data_counters/hll_redis_counter.rb +++ b/lib/gitlab/usage_data_counters/hll_redis_counter.rb @@ -1,5 +1,8 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module UsageDataCounters module HLLRedisCounter @@ -53,8 +56,6 @@ module Gitlab private def track(values, event_name, time: Time.zone.now) - return unless ::ServicePing::ServicePingSettings.enabled? - event = event_for(event_name) Gitlab::ErrorTracking.track_and_raise_for_dev_exception(UnknownEvent.new("Unknown event #{event_name}")) unless event.present? diff --git a/lib/gitlab/usage_data_counters/redis_counter.rb b/lib/gitlab/usage_data_counters/redis_counter.rb index 591e431c871..3f16681b642 100644 --- a/lib/gitlab/usage_data_counters/redis_counter.rb +++ b/lib/gitlab/usage_data_counters/redis_counter.rb @@ -1,17 +1,16 @@ # frozen_string_literal: true +# WARNING: This module has been deprecated and will be removed in the future +# Use InternalEvents.track_event instead https://docs.gitlab.com/ee/development/internal_analytics/internal_event_instrumentation/index.html + module Gitlab module UsageDataCounters module RedisCounter def increment(redis_counter_key) - return unless ::ServicePing::ServicePingSettings.enabled? - Gitlab::Redis::SharedState.with { |redis| redis.incr(redis_counter_key) } end def increment_by(redis_counter_key, incr) - return unless ::ServicePing::ServicePingSettings.enabled? - Gitlab::Redis::SharedState.with { |redis| redis.incrby(redis_counter_key, incr) } end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index f2db7e3c9b9..057e89a2a97 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -20,7 +20,7 @@ module Gitlab include JwtAuthenticatable class << self - def git_http_ok(repository, repo_type, user, action, show_all_refs: false) + def git_http_ok(repository, repo_type, user, action, show_all_refs: false, need_audit: false) raise "Unsupported action: #{action}" unless ALLOWED_GIT_HTTP_ACTIONS.include?(action.to_s) attrs = { @@ -28,6 +28,7 @@ module Gitlab GL_REPOSITORY: repo_type.identifier_for_container(repository.container), GL_USERNAME: user&.username, ShowAllRefs: show_all_refs, + NeedAudit: need_audit, Repository: repository.gitaly_repository.to_h, GitConfigOptions: [], GitalyServer: { |