diff options
| author | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
|---|---|---|
| committer | GitLab Bot <gitlab-bot@gitlab.com> | 2022-11-17 14:33:21 +0300 |
| commit | 7021455bd1ed7b125c55eb1b33c5a01f2bc55ee0 (patch) | |
| tree | 5bdc2229f5198d516781f8d24eace62fc7e589e9 /lib/gitlab | |
| parent | 185b095e93520f96e9cfc31d9c3e69b498cdab7c (diff) | |
Add latest changes from gitlab-org/gitlab@15-6-stable-eev15.6.0-rc42
Diffstat (limited to 'lib/gitlab')
310 files changed, 11117 insertions, 1838 deletions
diff --git a/lib/gitlab/application_context.rb b/lib/gitlab/application_context.rb index 1920e1443da..b6ad25e700b 100644 --- a/lib/gitlab/application_context.rb +++ b/lib/gitlab/application_context.rb @@ -11,6 +11,7 @@ module Gitlab LOG_KEY = Labkit::Context::LOG_KEY KNOWN_KEYS = [ :user, + :user_id, :project, :root_namespace, :client_id, @@ -98,6 +99,7 @@ module Gitlab assign_hash_if_value(hash, :artifacts_dependencies_count) hash[:user] = -> { username } if include_user? + hash[:user_id] = -> { user_id } if include_user? hash[:project] = -> { project_path } if include_project? hash[:root_namespace] = -> { root_namespace_path } if include_namespace? hash[:client_id] = -> { client } if include_client? @@ -147,6 +149,11 @@ module Gitlab associated_user&.username end + def user_id + associated_user = user || job_user + associated_user&.id + end + def root_namespace_path associated_routable = namespace || project || runner_project || runner_group || job_project associated_routable&.full_path_components&.first diff --git a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb index 7a68dd104a8..e8bdddca374 100644 --- a/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb +++ b/lib/gitlab/application_rate_limiter/increment_per_actioned_resource.rb @@ -10,7 +10,7 @@ module Gitlab def increment(cache_key, expiry) with_redis do |redis| redis.pipelined do |pipeline| - pipeline.sadd(cache_key, resource_key) + pipeline.sadd?(cache_key, resource_key) pipeline.expire(cache_key, expiry) pipeline.scard(cache_key) end.last diff --git a/lib/gitlab/asciidoc.rb b/lib/gitlab/asciidoc.rb index a9c2dd001cb..d55f2bc8ac9 100644 --- a/lib/gitlab/asciidoc.rb +++ b/lib/gitlab/asciidoc.rb @@ -2,6 +2,7 @@ require 'asciidoctor' require 'asciidoctor-plantuml' +require 'asciidoctor/extensions/asciidoctor_kroki/version' require 'asciidoctor/extensions/asciidoctor_kroki/extension' require 'asciidoctor/extensions' require 'gitlab/asciidoc/html5_converter' diff --git a/lib/gitlab/audit/type/definition.rb b/lib/gitlab/audit/type/definition.rb index af5dc9f4b44..f64f66f4ca4 100644 --- a/lib/gitlab/audit/type/definition.rb +++ b/lib/gitlab/audit/type/definition.rb @@ -5,6 +5,7 @@ module Gitlab module Type class Definition include ActiveModel::Validations + include ::Gitlab::Audit::Type::Shared attr_reader :path attr_reader :attributes @@ -17,18 +18,6 @@ module Gitlab AUDIT_EVENT_TYPE_SCHEMA_PATH = Rails.root.join('config', 'audit_events', 'types', 'type_schema.json') AUDIT_EVENT_TYPE_SCHEMA = JSONSchemer.schema(AUDIT_EVENT_TYPE_SCHEMA_PATH) - # The PARAMS in config/audit_events/types/type_schema.json - PARAMS = %i[ - name - description - introduced_by_issue - introduced_by_mr - group - milestone - saved_to_database - streamed - ].freeze - PARAMS.each do |param| define_method(param) do attributes[param] diff --git a/lib/gitlab/audit/type/shared.rb b/lib/gitlab/audit/type/shared.rb new file mode 100644 index 00000000000..999b7de13e2 --- /dev/null +++ b/lib/gitlab/audit/type/shared.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +# This file can contain only simple constructs as it is shared between: +# 1. `Pure Ruby`: `bin/audit-event-type` +# 2. `GitLab Rails`: `lib/gitlab/audit/type/definition.rb` + +module Gitlab + module Audit + module Type + module Shared + # The PARAMS in config/audit_events/types/type_schema.json + PARAMS = %i[ + name + description + introduced_by_issue + introduced_by_mr + group + milestone + saved_to_database + streamed + ].freeze + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb index 2ee0594d0a6..249c9d7af57 100644 --- a/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb +++ b/lib/gitlab/background_migration/backfill_cluster_agents_has_vulnerabilities.rb @@ -16,11 +16,10 @@ module Gitlab .where(has_vulnerabilities: false) end + operation_name :update_all + def perform - each_sub_batch( - operation_name: :update_all, - batching_scope: RELATION - ) do |sub_batch| + each_sub_batch(batching_scope: RELATION) do |sub_batch| sub_batch .joins(VULNERABILITY_READS_JOIN) .update_all(has_vulnerabilities: true) diff --git a/lib/gitlab/background_migration/backfill_group_features.rb b/lib/gitlab/background_migration/backfill_group_features.rb index 35b5282360f..4ea664e2529 100644 --- a/lib/gitlab/background_migration/backfill_group_features.rb +++ b/lib/gitlab/background_migration/backfill_group_features.rb @@ -5,10 +5,10 @@ module Gitlab # Backfill group_features for an array of groups class BackfillGroupFeatures < ::Gitlab::BackgroundMigration::BatchedMigrationJob job_arguments :batch_size + operation_name :upsert_group_features def perform each_sub_batch( - operation_name: :upsert_group_features, batching_arguments: { order_hint: :type }, batching_scope: ->(relation) { relation.where(type: 'Group') } ) do |sub_batch| diff --git a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb index b2d38ce6aa4..c95fed512c9 100644 --- a/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb +++ b/lib/gitlab/background_migration/backfill_imported_issue_search_data.rb @@ -9,10 +9,10 @@ module Gitlab class BackfillImportedIssueSearchData < BatchedMigrationJob SUB_BATCH_SIZE = 1_000 + operation_name :update_search_data + def perform - each_sub_batch( - operation_name: :update_search_data - ) do |sub_batch| + each_sub_batch do |sub_batch| update_search_data(sub_batch) rescue ActiveRecord::StatementInvalid => e raise unless e.cause.is_a?(PG::ProgramLimitExceeded) && e.message.include?('string is too long for tsvector') diff --git a/lib/gitlab/background_migration/backfill_internal_on_notes.rb b/lib/gitlab/background_migration/backfill_internal_on_notes.rb index 300f2cff6ca..fe05b4ec3c1 100644 --- a/lib/gitlab/background_migration/backfill_internal_on_notes.rb +++ b/lib/gitlab/background_migration/backfill_internal_on_notes.rb @@ -5,9 +5,10 @@ module Gitlab # This syncs the data to `internal` from `confidential` as we rename the column. class BackfillInternalOnNotes < BatchedMigrationJob scope_to -> (relation) { relation.where(confidential: true) } + operation_name :update_all def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(internal: true) end end diff --git a/lib/gitlab/background_migration/backfill_namespace_details.rb b/lib/gitlab/background_migration/backfill_namespace_details.rb index b8a51b576b6..640d9379351 100644 --- a/lib/gitlab/background_migration/backfill_namespace_details.rb +++ b/lib/gitlab/background_migration/backfill_namespace_details.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # Backfill namespace_details for a range of namespaces class BackfillNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_namespace_details + def perform - each_sub_batch(operation_name: :backfill_namespace_details) do |sub_batch| + each_sub_batch do |sub_batch| upsert_namespace_details(sub_batch) end end diff --git a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb index cd349bf3ae1..dca7f9fa921 100644 --- a/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb +++ b/lib/gitlab/background_migration/backfill_namespace_id_of_vulnerability_reads.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Sets the `namespace_id` of the existing `vulnerability_reads` records class BackfillNamespaceIdOfVulnerabilityReads < BatchedMigrationJob + operation_name :set_namespace_id + UPDATE_SQL = <<~SQL UPDATE vulnerability_reads @@ -16,7 +18,7 @@ module Gitlab SQL def perform - each_sub_batch(operation_name: :set_namespace_id) do |sub_batch| + each_sub_batch do |sub_batch| update_query = update_query_for(sub_batch) connection.execute(update_query) diff --git a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb index ce4c4a28b37..6520cd63711 100644 --- a/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb +++ b/lib/gitlab/background_migration/backfill_project_feature_package_registry_access_level.rb @@ -17,8 +17,10 @@ module Gitlab self.table_name = 'project_features' end + operation_name :update_all + def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| ProjectFeature.connection.execute( <<~SQL UPDATE project_features pf diff --git a/lib/gitlab/background_migration/backfill_project_import_level.rb b/lib/gitlab/background_migration/backfill_project_import_level.rb index 06706b729ea..21c239e0070 100644 --- a/lib/gitlab/background_migration/backfill_project_import_level.rb +++ b/lib/gitlab/background_migration/backfill_project_import_level.rb @@ -3,6 +3,8 @@ module Gitlab module BackgroundMigration class BackfillProjectImportLevel < BatchedMigrationJob + operation_name :update_import_level + LEVEL = { Gitlab::Access::NO_ACCESS => [0], Gitlab::Access::DEVELOPER => [2], @@ -11,7 +13,7 @@ module Gitlab }.freeze def perform - each_sub_batch(operation_name: :update_import_level) do |sub_batch| + each_sub_batch do |sub_batch| update_import_level(sub_batch) end end diff --git a/lib/gitlab/background_migration/backfill_project_namespace_details.rb b/lib/gitlab/background_migration/backfill_project_namespace_details.rb new file mode 100644 index 00000000000..9bee3cf21e8 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_namespace_details.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true +module Gitlab + module BackgroundMigration + # Backfill project namespace_details for a range of projects + class BackfillProjectNamespaceDetails < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :backfill_project_namespace_details + + def perform + each_sub_batch do |sub_batch| + upsert_project_namespace_details(sub_batch) + end + end + + def upsert_project_namespace_details(relation) + connection.execute( + <<~SQL + INSERT INTO namespace_details (description, description_html, cached_markdown_version, created_at, updated_at, namespace_id) + SELECT projects.description, projects.description_html, projects.cached_markdown_version, now(), now(), projects.project_namespace_id + FROM projects + WHERE projects.id IN(#{relation.select(:id).to_sql}) + ON CONFLICT (namespace_id) DO NOTHING; + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb index 815c346bb39..34dd3321125 100644 --- a/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb +++ b/lib/gitlab/background_migration/backfill_project_namespace_on_issues.rb @@ -4,21 +4,53 @@ module Gitlab module BackgroundMigration # Back-fills the `issues.namespace_id` by setting it to corresponding project.project_namespace_id class BackfillProjectNamespaceOnIssues < BatchedMigrationJob + MAX_UPDATE_RETRIES = 3 + + operation_name :update_all + def perform each_sub_batch( - operation_name: :update_all, batching_scope: -> (relation) { relation.joins("INNER JOIN projects ON projects.id = issues.project_id") .select("issues.id AS issue_id, projects.project_namespace_id").where(issues: { namespace_id: nil }) } ) do |sub_batch| - connection.execute <<~SQL + # updating issues table results in failed batches quite a bit, + # to prevent that as much as possible we try to update the same sub-batch up to 3 times. + update_with_retry(sub_batch) + end + end + + private + + # rubocop:disable Database/RescueQueryCanceled + # rubocop:disable Database/RescueStatementTimeout + def update_with_retry(sub_batch) + update_attempt = 1 + + begin + update_batch(sub_batch) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + sleep(5) + retry + end + + raise e + end + end + # rubocop:enable Database/RescueQueryCanceled + # rubocop:enable Database/RescueStatementTimeout + + def update_batch(sub_batch) + connection.execute <<~SQL UPDATE issues SET namespace_id = projects.project_namespace_id FROM (#{sub_batch.to_sql}) AS projects(issue_id, project_namespace_id) WHERE issues.id = issue_id - SQL - end + SQL end end end diff --git a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb b/lib/gitlab/background_migration/backfill_projects_with_coverage.rb deleted file mode 100644 index ca262c0bd59..00000000000 --- a/lib/gitlab/background_migration/backfill_projects_with_coverage.rb +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfill project_ci_feature_usages for a range of projects with coverage - class BackfillProjectsWithCoverage - class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation - self.table_name = 'project_ci_feature_usages' - end - - COVERAGE_ENUM_VALUE = 1 - INSERT_DELAY_SECONDS = 0.1 - - def perform(start_id, end_id, sub_batch_size) - report_results = ActiveRecord::Base.connection.execute <<~SQL - SELECT DISTINCT project_id, default_branch - FROM ci_daily_build_group_report_results - WHERE id BETWEEN #{start_id} AND #{end_id} - SQL - - report_results.to_a.in_groups_of(sub_batch_size, false) do |batch| - ProjectCiFeatureUsage.insert_all(build_values(batch)) - - sleep INSERT_DELAY_SECONDS - end - end - - private - - def build_values(batch) - batch.map do |data| - { - project_id: data['project_id'], - feature: COVERAGE_ENUM_VALUE, - default_branch: data['default_branch'] - } - end - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_user_details_fields.rb b/lib/gitlab/background_migration/backfill_user_details_fields.rb new file mode 100644 index 00000000000..8d8619256b0 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_user_details_fields.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Class that will backfill the following fields from user to user_details + # * linkedin + # * twitter + # * skype + # * website_url + # * location + # * organization + class BackfillUserDetailsFields < BatchedMigrationJob + operation_name :backfill_user_details_fields + + def perform + query = <<~SQL + (COALESCE(linkedin, '') IS DISTINCT FROM '') + OR (COALESCE(twitter, '') IS DISTINCT FROM '') + OR (COALESCE(skype, '') IS DISTINCT FROM '') + OR (COALESCE(website_url, '') IS DISTINCT FROM '') + OR (COALESCE(location, '') IS DISTINCT FROM '') + OR (COALESCE(organization, '') IS DISTINCT FROM '') + SQL + field_limit = UserDetail::DEFAULT_FIELD_LENGTH + + each_sub_batch( + batching_scope: ->(relation) { + relation.where(query).select( + 'id AS user_id', + "substring(COALESCE(linkedin, '') from 1 for #{field_limit}) AS linkedin", + "substring(COALESCE(twitter, '') from 1 for #{field_limit}) AS twitter", + "substring(COALESCE(skype, '') from 1 for #{field_limit}) AS skype", + "substring(COALESCE(website_url, '') from 1 for #{field_limit}) AS website_url", + "substring(COALESCE(location, '') from 1 for #{field_limit}) AS location", + "substring(COALESCE(organization, '') from 1 for #{field_limit}) AS organization" + ) + } + ) do |sub_batch| + upsert_user_details_fields(sub_batch) + end + end + + def upsert_user_details_fields(relation) + connection.execute( + <<~SQL + INSERT INTO user_details (user_id, linkedin, twitter, skype, website_url, location, organization) + #{relation.to_sql} + ON CONFLICT (user_id) + DO UPDATE SET + "linkedin" = EXCLUDED."linkedin", + "twitter" = EXCLUDED."twitter", + "skype" = EXCLUDED."skype", + "website_url" = EXCLUDED."website_url", + "location" = EXCLUDED."location", + "organization" = EXCLUDED."organization" + SQL + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb index 0c41d6af209..37b1a37569b 100644 --- a/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb +++ b/lib/gitlab/background_migration/backfill_vulnerability_reads_cluster_agent.rb @@ -4,6 +4,8 @@ module Gitlab module BackgroundMigration # Backfills the `vulnerability_reads.casted_cluster_agent_id` column class BackfillVulnerabilityReadsClusterAgent < Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :update_all + CLUSTER_AGENTS_JOIN = <<~SQL INNER JOIN cluster_agents ON CAST(vulnerability_reads.cluster_agent_id AS bigint) = cluster_agents.id AND @@ -15,7 +17,7 @@ module Gitlab scope_to ->(relation) { relation.where(report_type: CLUSTER_IMAGE_SCANNING_REPORT_TYPE) } def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch .joins(CLUSTER_AGENTS_JOIN) .update_all('casted_cluster_agent_id = CAST(vulnerability_reads.cluster_agent_id AS bigint)') diff --git a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb index 86d53ad798d..a020cabd1f4 100644 --- a/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb +++ b/lib/gitlab/background_migration/backfill_work_item_type_id_for_issues.rb @@ -19,10 +19,10 @@ module Gitlab } job_arguments :base_type, :base_type_id + operation_name :update_all def perform each_sub_batch( - operation_name: :update_all, batching_scope: -> (relation) { relation.where(work_item_type_id: nil) } ) do |sub_batch| first, last = sub_batch.pick(Arel.sql('min(id), max(id)')) diff --git a/lib/gitlab/background_migration/batched_migration_job.rb b/lib/gitlab/background_migration/batched_migration_job.rb index 11d15804344..64401bc0674 100644 --- a/lib/gitlab/background_migration/batched_migration_job.rb +++ b/lib/gitlab/background_migration/batched_migration_job.rb @@ -36,6 +36,12 @@ module Gitlab 0 end + def self.operation_name(operation) + define_method('operation_name') do + operation + end + end + def self.job_arguments(*args) args.each.with_index do |arg, index| define_method(arg) do @@ -70,7 +76,7 @@ module Gitlab attr_reader :start_id, :end_id, :batch_table, :batch_column, :sub_batch_size, :pause_ms, :connection - def each_sub_batch(operation_name: :default, batching_arguments: {}, batching_scope: nil) + def each_sub_batch(batching_arguments: {}, batching_scope: nil) all_batching_arguments = { column: batch_column, of: sub_batch_size }.merge(batching_arguments) relation = filter_batch(base_relation) @@ -85,7 +91,7 @@ module Gitlab end end - def distinct_each_batch(operation_name: :default, batching_arguments: {}) + def distinct_each_batch(batching_arguments: {}) if base_relation != filter_batch(base_relation) raise 'distinct_each_batch can not be used when additional filters are defined with scope_to' end @@ -111,6 +117,10 @@ module Gitlab batching_scope.call(relation) end + + def operation_name + raise('Operation name is required, please define it with `operation_name`') + end end end end diff --git a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb index 15e54431a44..136293242b2 100644 --- a/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb +++ b/lib/gitlab/background_migration/copy_column_using_background_migration_job.rb @@ -15,11 +15,12 @@ module Gitlab # We use the provided primary_key column to perform the update. class CopyColumnUsingBackgroundMigrationJob < BatchedMigrationJob job_arguments :copy_from, :copy_to + operation_name :update_all def perform assignment_clauses = build_assignment_clauses(copy_from, copy_to) - each_sub_batch(operation_name: :update_all) do |relation| + each_sub_batch do |relation| relation.update_all(assignment_clauses) end end diff --git a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb index c3e1019b72f..f93dcf83c49 100644 --- a/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb +++ b/lib/gitlab/background_migration/delete_orphaned_operational_vulnerabilities.rb @@ -16,13 +16,14 @@ module Gitlab ) SQL + operation_name :delete_orphaned_operational_vulnerabilities scope_to ->(relation) do relation .where(report_type: [REPORT_TYPES[:cluster_image_scanning], REPORT_TYPES[:custom]]) end def perform - each_sub_batch(operation_name: :delete_orphaned_operational_vulnerabilities) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch .where(NOT_EXISTS_SQL) .delete_all diff --git a/lib/gitlab/background_migration/destroy_invalid_group_members.rb b/lib/gitlab/background_migration/destroy_invalid_group_members.rb index 35ac42f76ab..9eb0d4489d6 100644 --- a/lib/gitlab/background_migration/destroy_invalid_group_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_group_members.rb @@ -9,8 +9,10 @@ module Gitlab .where(namespaces: { id: nil }) end + operation_name :delete_all + def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| invalid_ids = sub_batch.map(&:id) Gitlab::AppLogger.info({ message: 'Removing invalid group member records', deleted_count: invalid_ids.size, ids: invalid_ids }) diff --git a/lib/gitlab/background_migration/destroy_invalid_members.rb b/lib/gitlab/background_migration/destroy_invalid_members.rb index 7d78795bea9..17a141860ec 100644 --- a/lib/gitlab/background_migration/destroy_invalid_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_members.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration class DestroyInvalidMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(member_namespace_id: nil) } + operation_name :delete_all def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| deleted_members_data = sub_batch.map do |m| { id: m.id, source_id: m.source_id, source_type: m.source_type } end diff --git a/lib/gitlab/background_migration/destroy_invalid_project_members.rb b/lib/gitlab/background_migration/destroy_invalid_project_members.rb index 3c60f765c29..53b4712ef6e 100644 --- a/lib/gitlab/background_migration/destroy_invalid_project_members.rb +++ b/lib/gitlab/background_migration/destroy_invalid_project_members.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration class DestroyInvalidProjectMembers < Gitlab::BackgroundMigration::BatchedMigrationJob # rubocop:disable Style/Documentation scope_to ->(relation) { relation.where(source_type: 'Project') } + operation_name :delete_all def perform - each_sub_batch(operation_name: :delete_all) do |sub_batch| + each_sub_batch do |sub_batch| invalid_project_members = sub_batch .joins('LEFT OUTER JOIN projects ON members.source_id = projects.id') .where(projects: { id: nil }) diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb index 824054b31f2..b32e88581dd 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_licence_for_recent_public_projects.rb @@ -7,6 +7,8 @@ module Gitlab PUBLIC = 20 THRESHOLD_DATE = '2022-02-17 09:00:00' + operation_name :disable_legacy_open_source_licence_for_recent_public_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -14,7 +16,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_licence_for_recent_public_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC).where('created_at >= ?', THRESHOLD_DATE) } diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb index e759d504f8d..5685b782a71 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_inactive_public_projects.rb @@ -8,6 +8,8 @@ module Gitlab PUBLIC = 20 LAST_ACTIVITY_DATE = '2021-07-01' + operation_name :disable_legacy_open_source_license_available + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -15,7 +17,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_available, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC).where('last_activity_at < ?', LAST_ACTIVITY_DATE) } diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb index 019c3d15b3e..b5e5555bd2d 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_no_issues_no_repo_projects.rb @@ -6,6 +6,8 @@ module Gitlab class DisableLegacyOpenSourceLicenseForNoIssuesNoRepoProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :disable_legacy_open_source_license_for_no_issues_no_repo_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_for_no_issues_no_repo_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC) } ) do |sub_batch| no_issues_no_repo_projects = diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb index 3a9049b1f19..89863458676 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_one_member_no_repo_projects.rb @@ -6,6 +6,8 @@ module Gitlab class DisableLegacyOpenSourceLicenseForOneMemberNoRepoProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :disable_legacy_open_source_license_for_one_member_no_repo_projects + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :disable_legacy_open_source_license_for_one_member_no_repo_projects, batching_scope: ->(relation) { relation.where(visibility_level: PUBLIC) } ) do |sub_batch| one_member_no_repo_projects = diff --git a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb index 6e4d5d8ddcb..7d93f2d4fda 100644 --- a/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb +++ b/lib/gitlab/background_migration/disable_legacy_open_source_license_for_projects_less_than_one_mb.rb @@ -5,9 +5,10 @@ module Gitlab # Set `project_settings.legacy_open_source_license_available` to false for projects less than 1 MB class DisableLegacyOpenSourceLicenseForProjectsLessThanOneMb < ::Gitlab::BackgroundMigration::BatchedMigrationJob scope_to ->(relation) { relation.where(legacy_open_source_license_available: true) } + operation_name :disable_legacy_open_source_license_for_projects_less_than_one_mb def perform - each_sub_batch(operation_name: :disable_legacy_open_source_license_for_projects_less_than_one_mb) do |sub_batch| + each_sub_batch do |sub_batch| updates = { legacy_open_source_license_available: false, updated_at: Time.current } sub_batch diff --git a/lib/gitlab/background_migration/encrypt_static_object_token.rb b/lib/gitlab/background_migration/encrypt_static_object_token.rb index e1805d40bab..961dea028c9 100644 --- a/lib/gitlab/background_migration/encrypt_static_object_token.rb +++ b/lib/gitlab/background_migration/encrypt_static_object_token.rb @@ -40,8 +40,9 @@ module Gitlab encrypted_tokens_sql = user_encrypted_tokens.compact.map { |(id, token)| "(#{id}, '#{token}')" }.join(',') - if user_encrypted_tokens.present? - User.connection.execute(<<~SQL) + next unless user_encrypted_tokens.present? + + User.connection.execute(<<~SQL) WITH cte(cte_id, cte_token) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} ( SELECT * FROM (VALUES #{encrypted_tokens_sql}) AS t (id, token) @@ -50,8 +51,7 @@ module Gitlab SET static_object_token_encrypted = cte_token FROM cte WHERE cte_id = id - SQL - end + SQL end mark_job_as_succeeded(start_id, end_id) diff --git a/lib/gitlab/background_migration/expire_o_auth_tokens.rb b/lib/gitlab/background_migration/expire_o_auth_tokens.rb index 595e4ac9dc8..08bcdb8a789 100644 --- a/lib/gitlab/background_migration/expire_o_auth_tokens.rb +++ b/lib/gitlab/background_migration/expire_o_auth_tokens.rb @@ -4,9 +4,10 @@ module Gitlab module BackgroundMigration # Add expiry to all OAuth access tokens class ExpireOAuthTokens < ::Gitlab::BackgroundMigration::BatchedMigrationJob + operation_name :update_oauth_tokens + def perform each_sub_batch( - operation_name: :update_oauth_tokens, batching_scope: ->(relation) { relation.where(expires_in: nil) } ) do |sub_batch| update_oauth_tokens(sub_batch) diff --git a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb index 3f04e04fc4d..3dd867fa1fe 100644 --- a/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb +++ b/lib/gitlab/background_migration/populate_operation_visibility_permissions_from_operations.rb @@ -6,8 +6,10 @@ module Gitlab # monitor_access_level, deployments_access_level, infrastructure_access_level. # The operations_access_level setting is being split into three seperate toggles. class PopulateOperationVisibilityPermissionsFromOperations < BatchedMigrationJob + operation_name :populate_operations_visibility + def perform - each_sub_batch(operation_name: :populate_operations_visibility) do |batch| + each_sub_batch do |batch| batch.update_all('monitor_access_level=operations_access_level,' \ 'infrastructure_access_level=operations_access_level,' \ ' feature_flags_access_level=operations_access_level,'\ diff --git a/lib/gitlab/background_migration/populate_projects_star_count.rb b/lib/gitlab/background_migration/populate_projects_star_count.rb new file mode 100644 index 00000000000..085d576637e --- /dev/null +++ b/lib/gitlab/background_migration/populate_projects_star_count.rb @@ -0,0 +1,58 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # The class to populates the star counter of projects + class PopulateProjectsStarCount < BatchedMigrationJob + MAX_UPDATE_RETRIES = 3 + + operation_name :update_all + + def perform + each_sub_batch do |sub_batch| + update_with_retry(sub_batch) + end + end + + private + + # rubocop:disable Database/RescueQueryCanceled + # rubocop:disable Database/RescueStatementTimeout + def update_with_retry(sub_batch) + update_attempt = 1 + + begin + update_batch(sub_batch) + rescue ActiveRecord::StatementTimeout, ActiveRecord::QueryCanceled => e + update_attempt += 1 + + if update_attempt <= MAX_UPDATE_RETRIES + sleep(5) + retry + end + + raise e + end + end + # rubocop:enable Database/RescueQueryCanceled + # rubocop:enable Database/RescueStatementTimeout + + def update_batch(sub_batch) + ApplicationRecord.connection.execute <<~SQL + WITH batched_relation AS #{Gitlab::Database::AsWithMaterialized.materialized_if_supported} (#{sub_batch.select(:id).to_sql}) + UPDATE projects + SET star_count = ( + SELECT COUNT(*) + FROM users_star_projects + INNER JOIN users + ON users_star_projects.user_id = users.id + WHERE users_star_projects.project_id = batched_relation.id + AND users.state = 'active' + ) + FROM batched_relation + WHERE projects.id = batched_relation.id + SQL + end + end + end +end diff --git a/lib/gitlab/background_migration/recount_epic_cache_counts.rb b/lib/gitlab/background_migration/recount_epic_cache_counts.rb new file mode 100644 index 00000000000..42f84a33a5a --- /dev/null +++ b/lib/gitlab/background_migration/recount_epic_cache_counts.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Style/Documentation + class RecountEpicCacheCounts < Gitlab::BackgroundMigration::BatchedMigrationJob + def perform; end + end + # rubocop: enable Style/Documentation + end +end + +# rubocop: disable Layout/LineLength +# we just want to re-enqueue the previous BackfillEpicCacheCounts migration, +# because it's a EE-only migation and it's a module, we just prepend new +# RecountEpicCacheCounts with existing batched migration module (which is same in both cases) +Gitlab::BackgroundMigration::RecountEpicCacheCounts.prepend_mod_with('Gitlab::BackgroundMigration::BackfillEpicCacheCounts') +# rubocop: enable Layout/LineLength diff --git a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb index d30263976e8..dc7c16d7947 100644 --- a/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb +++ b/lib/gitlab/background_migration/remove_backfilled_job_artifacts_expire_at.rb @@ -6,6 +6,8 @@ module Gitlab # https://gitlab.com/gitlab-org/gitlab/-/merge_requests/47723. # These job artifacts will not be deleted and will have their `expire_at` removed. class RemoveBackfilledJobArtifactsExpireAt < BatchedMigrationJob + operation_name :update_all + # The migration would have backfilled `expire_at` # to midnight on the 22nd of the month of the local timezone, # storing it as UTC time in the database. @@ -32,9 +34,7 @@ module Gitlab } def perform - each_sub_batch( - operation_name: :update_all - ) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(expire_at: nil) end end diff --git a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb index 5b1d630bb03..a284c04d4f5 100644 --- a/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb +++ b/lib/gitlab/background_migration/remove_self_managed_wiki_notes.rb @@ -4,10 +4,10 @@ module Gitlab module BackgroundMigration # Removes obsolete wiki notes class RemoveSelfManagedWikiNotes < BatchedMigrationJob + operation_name :delete_all + def perform - each_sub_batch( - operation_name: :delete_all - ) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.where(noteable_type: 'Wiki').delete_all end end diff --git a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb index 718fb0aaa71..1b13c2ab7ef 100644 --- a/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb +++ b/lib/gitlab/background_migration/rename_task_system_note_to_checklist_item.rb @@ -13,8 +13,10 @@ module Gitlab relation.where(system_note_metadata: { action: :task }) } + operation_name :update_all + def perform - each_sub_batch(operation_name: :update_all) do |sub_batch| + each_sub_batch do |sub_batch| ApplicationRecord.connection.execute <<~SQL UPDATE notes SET note = REGEXP_REPLACE(notes.note,'#{REPLACE_REGEX}', '#{TEXT_REPLACEMENT}') diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb index 952f3b0e3c3..832385fd662 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_encrypted_values.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # A job to nullify duplicate token_encrypted values in ci_runners table in batches class ResetDuplicateCiRunnersTokenEncryptedValues < BatchedMigrationJob + operation_name :nullify_duplicate_ci_runner_token_encrypted_values + def perform - each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_encrypted_values) do |sub_batch| + each_sub_batch do |sub_batch| # Reset duplicate runner encrypted tokens that would prevent creating an unique index. nullify_duplicate_ci_runner_token_encrypted_values(sub_batch) end diff --git a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb index cfd6a4e4091..5f552accd8d 100644 --- a/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb +++ b/lib/gitlab/background_migration/reset_duplicate_ci_runners_token_values.rb @@ -4,8 +4,10 @@ module Gitlab module BackgroundMigration # A job to nullify duplicate token values in ci_runners table in batches class ResetDuplicateCiRunnersTokenValues < BatchedMigrationJob + operation_name :nullify_duplicate_ci_runner_token_values + def perform - each_sub_batch(operation_name: :nullify_duplicate_ci_runner_token_values) do |sub_batch| + each_sub_batch do |sub_batch| # Reset duplicate runner tokens that would prevent creating an unique index. nullify_duplicate_ci_runner_token_values(sub_batch) end diff --git a/lib/gitlab/background_migration/sanitize_confidential_todos.rb b/lib/gitlab/background_migration/sanitize_confidential_todos.rb new file mode 100644 index 00000000000..d3ef6ac3019 --- /dev/null +++ b/lib/gitlab/background_migration/sanitize_confidential_todos.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # Iterates through confidential notes and removes any its todos if user can + # not read the note + # + # Warning: This migration is not properly isolated. The reason for this is + # that we need to check permission for notes and it would be difficult + # to extract all related logic. + # Details in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/87908#note_952459215 + class SanitizeConfidentialTodos < BatchedMigrationJob + scope_to ->(relation) { relation.where(confidential: true) } + + operation_name :delete_invalid_todos + + def perform + each_sub_batch do |sub_batch| + delete_ids = invalid_todo_ids(sub_batch) + + Todo.where(id: delete_ids).delete_all if delete_ids.present? + end + end + + private + + def invalid_todo_ids(notes_batch) + todos = Todo.where(note_id: notes_batch.select(:id)).includes(:note, :user) + + todos.each_with_object([]) do |todo, ids| + ids << todo.id if invalid_todo?(todo) + end + end + + def invalid_todo?(todo) + return false unless todo.note + return false if Ability.allowed?(todo.user, :read_todo, todo) + + logger.info( + message: "#{self.class.name} deleting invalid todo", + attributes: todo.attributes + ) + + true + end + + def logger + @logger ||= Gitlab::BackgroundMigration::Logger.build + end + end + end +end diff --git a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb index a0cfeed618a..dfd71bb8b5f 100644 --- a/lib/gitlab/background_migration/set_correct_vulnerability_state.rb +++ b/lib/gitlab/background_migration/set_correct_vulnerability_state.rb @@ -7,9 +7,10 @@ module Gitlab DISMISSED_STATE = 2 scope_to ->(relation) { relation.where.not(dismissed_at: nil) } + operation_name :update_vulnerabilities_state def perform - each_sub_batch(operation_name: :update_vulnerabilities_state) do |sub_batch| + each_sub_batch do |sub_batch| sub_batch.update_all(state: DISMISSED_STATE) end end diff --git a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb index e85b1bc402a..4ae7ad897cf 100644 --- a/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb +++ b/lib/gitlab/background_migration/set_legacy_open_source_license_available_for_non_public_projects.rb @@ -6,6 +6,8 @@ module Gitlab class SetLegacyOpenSourceLicenseAvailableForNonPublicProjects < ::Gitlab::BackgroundMigration::BatchedMigrationJob PUBLIC = 20 + operation_name :set_legacy_open_source_license_available + # Migration only version of `project_settings` table class ProjectSetting < ApplicationRecord self.table_name = 'project_settings' @@ -13,7 +15,6 @@ module Gitlab def perform each_sub_batch( - operation_name: :set_legacy_open_source_license_available, batching_scope: ->(relation) { relation.where.not(visibility_level: PUBLIC) } ) do |sub_batch| ProjectSetting.where(project_id: sub_batch).update_all(legacy_open_source_license_available: false) diff --git a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb index 04a2ceebef8..b2cf8298e4f 100644 --- a/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb +++ b/lib/gitlab/background_migration/update_delayed_project_removal_to_null_for_user_namespaces.rb @@ -10,10 +10,10 @@ module Gitlab self.table_name = 'namespace_settings' end + operation_name :set_delayed_project_removal_to_null_for_user_namespace + def perform - each_sub_batch( - operation_name: :set_delayed_project_removal_to_null_for_user_namespace - ) do |sub_batch| + each_sub_batch do |sub_batch| set_delayed_project_removal_to_null_for_user_namespace(sub_batch) end end diff --git a/lib/gitlab/cache/ci/project_pipeline_status.rb b/lib/gitlab/cache/ci/project_pipeline_status.rb index 9209c9b4927..b2630a7ad7a 100644 --- a/lib/gitlab/cache/ci/project_pipeline_status.rb +++ b/lib/gitlab/cache/ci/project_pipeline_status.rb @@ -85,7 +85,7 @@ module Gitlab end def load_from_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) self.status = nil if self.status.empty? @@ -93,13 +93,13 @@ module Gitlab end def store_in_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) end end def delete_from_cache - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.del(cache_key) end end @@ -107,7 +107,7 @@ module Gitlab def has_cache? return self.loaded unless self.loaded.nil? - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.exists?(cache_key) # rubocop:disable CodeReuse/ActiveRecord end end @@ -125,6 +125,10 @@ module Gitlab project.commit end end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/cache/import/caching.rb b/lib/gitlab/cache/import/caching.rb index 4e7a7f326a5..7fec6584ba3 100644 --- a/lib/gitlab/cache/import/caching.rb +++ b/lib/gitlab/cache/import/caching.rb @@ -33,7 +33,7 @@ module Gitlab # timeout - The new timeout of the key if the key is to be refreshed. def self.read(raw_key, timeout: TIMEOUT) key = cache_key_for(raw_key) - value = Redis::Cache.with { |redis| redis.get(key) } + value = with_redis { |redis| redis.get(key) } if value.present? # We refresh the expiration time so frequently used keys stick @@ -44,7 +44,7 @@ module Gitlab # did not find a matching GitLab user. In that case we _don't_ want to # refresh the TTL so we automatically pick up the right data when said # user were to register themselves on the GitLab instance. - Redis::Cache.with { |redis| redis.expire(key, timeout) } + with_redis { |redis| redis.expire(key, timeout) } end value @@ -69,7 +69,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.set(key, value, ex: timeout) end @@ -85,7 +85,7 @@ module Gitlab def self.increment(raw_key, timeout: TIMEOUT) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| value = redis.incr(key) redis.expire(key, timeout) @@ -105,7 +105,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.incrby(key, value) redis.expire(key, timeout) end @@ -121,9 +121,9 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| - m.sadd(key, value) + m.sadd?(key, value) m.expire(key, timeout) end end @@ -149,7 +149,7 @@ module Gitlab def self.values_from_set(raw_key) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.smembers(key) end end @@ -160,14 +160,16 @@ module Gitlab # key_prefix - prefix inserted before each key # timeout - The time after which the cache key should expire. def self.write_multiple(mapping, key_prefix: nil, timeout: TIMEOUT) - Redis::Cache.with do |redis| - redis.pipelined do |multi| - mapping.each do |raw_key, value| - key = cache_key_for("#{key_prefix}#{raw_key}") + with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.pipelined do |multi| + mapping.each do |raw_key, value| + key = cache_key_for("#{key_prefix}#{raw_key}") - validate_redis_value!(value) + validate_redis_value!(value) - multi.set(key, value, ex: timeout) + multi.set(key, value, ex: timeout) + end end end end @@ -180,7 +182,7 @@ module Gitlab def self.expire(raw_key, timeout) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.expire(key, timeout) end end @@ -199,7 +201,7 @@ module Gitlab validate_redis_value!(value) key = cache_key_for(raw_key) - val = Redis::Cache.with do |redis| + val = with_redis do |redis| redis .eval(WRITE_IF_GREATER_SCRIPT, keys: [key], argv: [value, timeout]) end @@ -218,7 +220,7 @@ module Gitlab key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| m.hset(key, field, value) m.expire(key, timeout) @@ -232,7 +234,7 @@ module Gitlab def self.values_from_hash(raw_key) key = cache_key_for(raw_key) - Redis::Cache.with do |redis| + with_redis do |redis| redis.hgetall(key) end end @@ -241,6 +243,10 @@ module Gitlab "#{Redis::Cache::CACHE_NAMESPACE}:#{raw_key}" end + def self.with_redis(&block) + Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end + def self.validate_redis_value!(value) value_as_string = value.to_s return if value_as_string.is_a?(String) diff --git a/lib/gitlab/cache/metrics.rb b/lib/gitlab/cache/metrics.rb new file mode 100644 index 00000000000..0143052beb1 --- /dev/null +++ b/lib/gitlab/cache/metrics.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +# Instrumentation for cache efficiency metrics +module Gitlab + module Cache + class Metrics + DEFAULT_BUCKETS = [0, 1, 5].freeze + VALID_BACKING_RESOURCES = [:cpu, :database, :gitaly, :memory, :unknown].freeze + DEFAULT_BACKING_RESOURCE = :unknown + + def initialize( + caller_id:, + cache_identifier:, + feature_category: ::Gitlab::FeatureCategories::FEATURE_CATEGORY_DEFAULT, + backing_resource: DEFAULT_BACKING_RESOURCE + ) + @caller_id = caller_id + @cache_identifier = cache_identifier + @feature_category = Gitlab::FeatureCategories.default.get!(feature_category) + @backing_resource = fetch_backing_resource!(backing_resource) + end + + # Increase cache hit counter + # + def increment_cache_hit + counter.increment(labels.merge(cache_hit: true)) + end + + # Increase cache miss counter + # + def increment_cache_miss + counter.increment(labels.merge(cache_hit: false)) + end + + # Measure the duration of cacheable action + # + # @example + # observe_cache_generation do + # cacheable_action + # end + # + def observe_cache_generation(&block) + real_start = Gitlab::Metrics::System.monotonic_time + + value = yield + + histogram.observe({}, Gitlab::Metrics::System.monotonic_time - real_start) + + value + end + + private + + attr_reader :caller_id, :cache_identifier, :feature_category, :backing_resource + + def counter + @counter ||= Gitlab::Metrics.counter(:redis_hit_miss_operations_total, "Hit/miss Redis cache counter") + end + + def histogram + @histogram ||= Gitlab::Metrics.histogram( + :redis_cache_generation_duration_seconds, + 'Duration of Redis cache generation', + labels, + DEFAULT_BUCKETS + ) + end + + def labels + @labels ||= { + caller_id: caller_id, + cache_identifier: cache_identifier, + feature_category: feature_category, + backing_resource: backing_resource + } + end + + def fetch_backing_resource!(resource) + return resource if VALID_BACKING_RESOURCES.include?(resource) + + raise "Unknown backing resource: #{resource}" if Gitlab.dev_or_test_env? + + DEFAULT_BACKING_RESOURCE + end + end + end +end diff --git a/lib/gitlab/ci/ansi2html.rb b/lib/gitlab/ci/ansi2html.rb index 2ab702aa4f9..19819ff7275 100644 --- a/lib/gitlab/ci/ansi2html.rb +++ b/lib/gitlab/ci/ansi2html.rb @@ -312,9 +312,10 @@ module Gitlab normalized_section = section_to_class_name(section) - if action == "start" + case action + when "start" handle_section_start(normalized_section, timestamp) - elsif action == "end" + when "end" handle_section_end(normalized_section, timestamp) end end diff --git a/lib/gitlab/ci/ansi2json/converter.rb b/lib/gitlab/ci/ansi2json/converter.rb index ddf40296809..78f6c5bf0aa 100644 --- a/lib/gitlab/ci/ansi2json/converter.rb +++ b/lib/gitlab/ci/ansi2json/converter.rb @@ -107,9 +107,10 @@ module Gitlab section_name = sanitize_section_name(section) - if action == 'start' + case action + when 'start' handle_section_start(scanner, section_name, timestamp, options) - elsif action == 'end' + when 'end' handle_section_end(scanner, section_name, timestamp) else raise 'unsupported action' diff --git a/lib/gitlab/ci/build/image.rb b/lib/gitlab/ci/build/image.rb index 7dc375e05eb..84f8eae8deb 100644 --- a/lib/gitlab/ci/build/image.rb +++ b/lib/gitlab/ci/build/image.rb @@ -24,10 +24,11 @@ module Gitlab end def initialize(image) - if image.is_a?(String) + case image + when String @name = image @ports = [] - elsif image.is_a?(Hash) + when Hash @alias = image[:alias] @command = image[:command] @entrypoint = image[:entrypoint] diff --git a/lib/gitlab/ci/build/rules/rule/clause/exists.rb b/lib/gitlab/ci/build/rules/rule/clause/exists.rb index aebd81e7b07..c55615bb83b 100644 --- a/lib/gitlab/ci/build/rules/rule/clause/exists.rb +++ b/lib/gitlab/ci/build/rules/rule/clause/exists.rb @@ -9,20 +9,30 @@ module Gitlab MAX_PATTERN_COMPARISONS = 10_000 def initialize(globs) - globs = Array(globs) - - @top_level_only = globs.all?(&method(:top_level_glob?)) - @exact_globs, @pattern_globs = globs.partition(&method(:exact_glob?)) + @globs = Array(globs) + @top_level_only = @globs.all?(&method(:top_level_glob?)) end def satisfied_by?(_pipeline, context) paths = worktree_paths(context) + exact_globs, pattern_globs = separate_globs(context) - exact_matches?(paths) || pattern_matches?(paths) + exact_matches?(paths, exact_globs) || pattern_matches?(paths, pattern_globs) end private + def separate_globs(context) + expanded_globs = expand_globs(context) + expanded_globs.partition(&method(:exact_glob?)) + end + + def expand_globs(context) + @globs.map do |glob| + ExpandVariables.expand_existing(glob, -> { context.variables_hash }) + end + end + def worktree_paths(context) return [] unless context.project @@ -33,13 +43,16 @@ module Gitlab end end - def exact_matches?(paths) - @exact_globs.any? { |glob| paths.bsearch { |path| glob <=> path } } + def exact_matches?(paths, exact_globs) + exact_globs.any? do |glob| + paths.bsearch { |path| glob <=> path } + end end - def pattern_matches?(paths) + def pattern_matches?(paths, pattern_globs) comparisons = 0 - @pattern_globs.any? do |glob| + + pattern_globs.any? do |glob| paths.any? do |path| comparisons += 1 comparisons > MAX_PATTERN_COMPARISONS || pattern_match?(glob, path) diff --git a/lib/gitlab/ci/config.rb b/lib/gitlab/ci/config.rb index 661c6fb87e3..ee537f4efe5 100644 --- a/lib/gitlab/ci/config.rb +++ b/lib/gitlab/ci/config.rb @@ -73,6 +73,10 @@ module Gitlab root.variables_entry.value_with_data end + def variables_with_prefill_data + root.variables_entry.value_with_prefill_data + end + def stages root.stages_value end diff --git a/lib/gitlab/ci/config/entry/bridge.rb b/lib/gitlab/ci/config/entry/bridge.rb index 73742298628..ee99354cb28 100644 --- a/lib/gitlab/ci/config/entry/bridge.rb +++ b/lib/gitlab/ci/config/entry/bridge.rb @@ -18,7 +18,7 @@ module Gitlab validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS with_options allow_nil: true do - validates :when, inclusion: { + validates :when, type: String, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 7513936a18a..8e7f6ba4326 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -21,7 +21,7 @@ module Gitlab validates :script, presence: true with_options allow_nil: true do - validates :when, inclusion: { + validates :when, type: String, inclusion: { in: ALLOWED_WHEN, message: "should be one of: #{ALLOWED_WHEN.join(', ')}" } diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index 2d2032b1d8c..e0a052ffdfd 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -60,6 +60,7 @@ module Gitlab entry :variables, ::Gitlab::Ci::Config::Entry::Variables, description: 'Environment variables available for this job.', + metadata: { allowed_value_data: %i[value expand] }, inherit: false entry :inherit, ::Gitlab::Ci::Config::Entry::Inherit, diff --git a/lib/gitlab/ci/config/entry/root.rb b/lib/gitlab/ci/config/entry/root.rb index 1d7d8617c74..a30e6a0d9c3 100644 --- a/lib/gitlab/ci/config/entry/root.rb +++ b/lib/gitlab/ci/config/entry/root.rb @@ -50,7 +50,7 @@ module Gitlab entry :variables, Entry::Variables, description: 'Environment variables that will be used.', - metadata: { allowed_value_data: %i[value description], allow_array_value: true }, + metadata: { allowed_value_data: %i[value description expand], allow_array_value: true }, reserved: true entry :stages, Entry::Stages, diff --git a/lib/gitlab/ci/config/entry/variable.rb b/lib/gitlab/ci/config/entry/variable.rb index 54c153c8b07..16091758916 100644 --- a/lib/gitlab/ci/config/entry/variable.rb +++ b/lib/gitlab/ci/config/entry/variable.rb @@ -33,6 +33,10 @@ module Gitlab def value_with_data { value: @config.to_s } end + + def value_with_prefill_data + value_with_data + end end class ComplexVariable < ::Gitlab::Config::Entry::Node @@ -48,6 +52,9 @@ module Gitlab validates :key, alphanumeric: true validates :config_value, alphanumeric: true, allow_nil: false, if: :config_value_defined? validates :config_description, alphanumeric: true, allow_nil: false, if: :config_description_defined? + validates :config_expand, boolean: true, + allow_nil: false, + if: -> { ci_raw_variables_in_yaml_config_enabled? && config_expand_defined? } validate do allowed_value_data = Array(opt(:allowed_value_data)) @@ -67,7 +74,22 @@ module Gitlab end def value_with_data - { value: value, description: config_description }.compact + if ci_raw_variables_in_yaml_config_enabled? + { + value: value, + raw: (!config_expand if config_expand_defined?) + }.compact + else + { + value: value + }.compact + end + end + + def value_with_prefill_data + value_with_data.merge( + description: config_description + ).compact end def config_value @@ -78,6 +100,10 @@ module Gitlab @config[:description] end + def config_expand + @config[:expand] + end + def config_value_defined? config.key?(:value) end @@ -85,6 +111,14 @@ module Gitlab def config_description_defined? config.key?(:description) end + + def config_expand_defined? + config.key?(:expand) + end + + def ci_raw_variables_in_yaml_config_enabled? + YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) + end end class ComplexArrayVariable < ComplexVariable @@ -110,8 +144,10 @@ module Gitlab config_value.first end - def value_with_data - super.merge(value_options: config_value).compact + def value_with_prefill_data + super.merge( + value_options: config_value + ).compact end end diff --git a/lib/gitlab/ci/config/entry/variables.rb b/lib/gitlab/ci/config/entry/variables.rb index 4430a11dda7..ef4f74b9f56 100644 --- a/lib/gitlab/ci/config/entry/variables.rb +++ b/lib/gitlab/ci/config/entry/variables.rb @@ -29,6 +29,12 @@ module Gitlab end end + def value_with_prefill_data + @entries.to_h do |key, entry| + [key.to_s, entry.value_with_prefill_data] + end + end + private def composable_class(_name, _config) diff --git a/lib/gitlab/ci/config/external/file/artifact.rb b/lib/gitlab/ci/config/external/file/artifact.rb index 1244c7f7475..21a57640aee 100644 --- a/lib/gitlab/ci/config/external/file/artifact.rb +++ b/lib/gitlab/ci/config/external/file/artifact.rb @@ -42,29 +42,20 @@ module Gitlab context&.parent_pipeline&.project end - def validate_content! - return unless ensure_preconditions_satisfied! - - errors.push("File `#{masked_location}` is empty!") unless content.present? - end - - def ensure_preconditions_satisfied! - unless creating_child_pipeline? - errors.push('Including configs from artifacts is only allowed when triggering child pipelines') - return false - end - - unless job_name.present? - errors.push("Job must be provided when including configs from artifacts") - return false - end - - unless artifact_job.present? - errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") - return false + def validate_context! + context.logger.instrument(:config_file_artifact_validate_context) do + if !creating_child_pipeline? + errors.push('Including configs from artifacts is only allowed when triggering child pipelines') + elsif !job_name.present? + errors.push("Job must be provided when including configs from artifacts") + elsif !artifact_job.present? + errors.push("Job `#{masked_job_name}` not found in parent pipeline or does not have artifacts!") + end end + end - true + def validate_content! + errors.push("File `#{masked_location}` is empty!") unless content.present? end def artifact_job diff --git a/lib/gitlab/ci/config/external/file/base.rb b/lib/gitlab/ci/config/external/file/base.rb index 89da0796906..57ff606c9ee 100644 --- a/lib/gitlab/ci/config/external/file/base.rb +++ b/lib/gitlab/ci/config/external/file/base.rb @@ -47,12 +47,11 @@ module Gitlab end def validate! - context.logger.instrument(:config_file_validation) do - validate_execution_time! - validate_location! - validate_content! if errors.none? - validate_hash! if errors.none? - end + validate_execution_time! + validate_location! + validate_context! if valid? + fetch_and_validate_content! if valid? + load_and_validate_expanded_hash! if valid? end def metadata @@ -100,6 +99,34 @@ module Gitlab end end + def validate_context! + raise NotImplementedError, 'subclass must implement validate_context' + end + + def fetch_and_validate_content! + context.logger.instrument(:config_file_fetch_content) do + content # calling the method fetches then memoizes the result + end + + return if errors.any? + + context.logger.instrument(:config_file_validate_content) do + validate_content! + end + end + + def load_and_validate_expanded_hash! + context.logger.instrument(:config_file_fetch_content_hash) do + content_hash # calling the method loads then memoizes the result + end + + context.logger.instrument(:config_file_expand_content_includes) do + expanded_content_hash # calling the method expands then memoizes the result + end + + validate_hash! + end + def validate_content! if content.blank? errors.push("Included file `#{masked_location}` is empty or does not exist!") diff --git a/lib/gitlab/ci/config/external/file/local.rb b/lib/gitlab/ci/config/external/file/local.rb index 36fc5c656fc..0912a732158 100644 --- a/lib/gitlab/ci/config/external/file/local.rb +++ b/lib/gitlab/ci/config/external/file/local.rb @@ -31,10 +31,14 @@ module Gitlab private + def validate_context! + return if context.project&.repository + + errors.push("Local file `#{masked_location}` does not have project!") + end + def validate_content! - if context.project&.repository.nil? - errors.push("Local file `#{masked_location}` does not have project!") - elsif content.nil? + if content.nil? errors.push("Local file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Local file `#{masked_location}` is empty!") diff --git a/lib/gitlab/ci/config/external/file/project.rb b/lib/gitlab/ci/config/external/file/project.rb index 89418bd6a21..553cbd819ad 100644 --- a/lib/gitlab/ci/config/external/file/project.rb +++ b/lib/gitlab/ci/config/external/file/project.rb @@ -39,12 +39,16 @@ module Gitlab private - def validate_content! + def validate_context! if !can_access_local_content? errors.push("Project `#{masked_project_name}` not found or access denied! Make sure any includes in the pipeline configuration are correctly defined.") elsif sha.nil? errors.push("Project `#{masked_project_name}` reference `#{masked_ref_name}` does not exist!") - elsif content.nil? + end + end + + def validate_content! + if content.nil? errors.push("Project `#{masked_project_name}` file `#{masked_location}` does not exist!") elsif content.blank? errors.push("Project `#{masked_project_name}` file `#{masked_location}` is empty!") @@ -58,7 +62,11 @@ module Gitlab end def can_access_local_content? - Ability.allowed?(context.user, :download_code, project) + strong_memoize(:can_access_local_content) do + context.logger.instrument(:config_file_project_validate_access) do + Ability.allowed?(context.user, :download_code, project) + end + end end def fetch_local_content diff --git a/lib/gitlab/ci/config/external/file/remote.rb b/lib/gitlab/ci/config/external/file/remote.rb index 3984bf9e4f8..b0c540685d4 100644 --- a/lib/gitlab/ci/config/external/file/remote.rb +++ b/lib/gitlab/ci/config/external/file/remote.rb @@ -30,6 +30,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/ci/config/external/file/template.rb b/lib/gitlab/ci/config/external/file/template.rb index 5fcf7c71bdf..53236cb317b 100644 --- a/lib/gitlab/ci/config/external/file/template.rb +++ b/lib/gitlab/ci/config/external/file/template.rb @@ -33,6 +33,10 @@ module Gitlab private + def validate_context! + # no-op + end + def validate_location! super diff --git a/lib/gitlab/ci/config/external/mapper.rb b/lib/gitlab/ci/config/external/mapper.rb index 2a1060a6059..fc03ac125fd 100644 --- a/lib/gitlab/ci/config/external/mapper.rb +++ b/lib/gitlab/ci/config/external/mapper.rb @@ -8,13 +8,15 @@ module Gitlab include Gitlab::Utils::StrongMemoize FILE_CLASSES = [ - External::File::Remote, - External::File::Template, External::File::Local, External::File::Project, + External::File::Remote, + External::File::Template, External::File::Artifact ].freeze + FILE_SUBKEYS = FILE_CLASSES.map { |f| f.name.demodulize.downcase }.freeze + Error = Class.new(StandardError) AmbigiousSpecificationError = Class.new(Error) TooManyIncludesError = Class.new(Error) @@ -120,9 +122,13 @@ module Gitlab file_class.new(location, context) end.select(&:matching?) - raise AmbigiousSpecificationError, "Include `#{masked_location(location.to_json)}` needs to match exactly one accessor!" unless matching.one? - - matching.first + if matching.one? + matching.first + elsif matching.empty? + raise AmbigiousSpecificationError, "`#{masked_location(location.to_json)}` does not have a valid subkey for include. Valid subkeys are: `#{FILE_SUBKEYS.join('`, `')}`" + else + raise AmbigiousSpecificationError, "Each include must use only one of: `#{FILE_SUBKEYS.join('`, `')}`" + end end def verify!(location_object) diff --git a/lib/gitlab/ci/parsers/codequality/code_climate.rb b/lib/gitlab/ci/parsers/codequality/code_climate.rb index 628d50b84cb..14ea907edd8 100644 --- a/lib/gitlab/ci/parsers/codequality/code_climate.rb +++ b/lib/gitlab/ci/parsers/codequality/code_climate.rb @@ -5,23 +5,36 @@ module Gitlab module Parsers module Codequality class CodeClimate - def parse!(json_data, codequality_report) + def parse!(json_data, codequality_report, metadata = {}) root = Gitlab::Json.parse(json_data) - parse_all(root, codequality_report) + parse_all(root, codequality_report, metadata) rescue JSON::ParserError => e codequality_report.set_error_message("JSON parsing failed: #{e}") end private - def parse_all(root, codequality_report) + def parse_all(root, codequality_report, metadata) return unless root.present? root.each do |degradation| - break unless codequality_report.add_degradation(degradation) + break unless codequality_report.valid_degradation?(degradation) + + degradation['web_url'] = web_url(degradation, metadata) + codequality_report.add_degradation(degradation) end end + + def web_url(degradation, metadata) + return unless metadata[:project].present? && metadata[:commit_sha].present? + + path = degradation.dig('location', 'path') + line = degradation.dig('location', 'lines', 'begin') || + degradation.dig('location', 'positions', 'begin', 'line') + "#{Routing.url_helpers.project_blob_url( + metadata[:project], File.join(metadata[:commit_sha], path))}#L#{line}" + end end end end diff --git a/lib/gitlab/ci/parsers/coverage/sax_document.rb b/lib/gitlab/ci/parsers/coverage/sax_document.rb index 27cce0e3a3b..ddd9c80f5ea 100644 --- a/lib/gitlab/ci/parsers/coverage/sax_document.rb +++ b/lib/gitlab/ci/parsers/coverage/sax_document.rb @@ -76,7 +76,12 @@ module Gitlab # | /builds/foo/test/something | something | # | /builds/foo/test/ | nil | # | /builds/foo/test | nil | - node.split("#{project_path}/", 2)[1] + # | D:\builds\foo\bar\app\ | app\ | + unixify(node).split("#{project_path}/", 2)[1] + end + + def unixify(path) + path.tr('\\', '/') end def remove_matched_filenames diff --git a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb index aa594ca4049..bc62fbe55ec 100644 --- a/lib/gitlab/ci/parsers/sbom/cyclonedx.rb +++ b/lib/gitlab/ci/parsers/sbom/cyclonedx.rb @@ -61,23 +61,19 @@ module Gitlab end def parse_components - data['components']&.each do |component_data| - type = component_data['type'] - next unless supported_component_type?(type) - + data['components']&.each_with_index do |component_data, index| component = ::Gitlab::Ci::Reports::Sbom::Component.new( - type: type, + type: component_data['type'], name: component_data['name'], + purl: component_data['purl'], version: component_data['version'] ) - report.add_component(component) + report.add_component(component) if component.ingestible? + rescue ::Sbom::PackageUrl::InvalidPackageUrl + report.add_error("/components/#{index}/purl is invalid") end end - - def supported_component_type?(type) - ::Enums::Sbom.component_types.include?(type.to_sym) - end end end end diff --git a/lib/gitlab/ci/parsers/security/common.rb b/lib/gitlab/ci/parsers/security/common.rb index 0c117d5f214..0ac012b9fd1 100644 --- a/lib/gitlab/ci/parsers/security/common.rb +++ b/lib/gitlab/ci/parsers/security/common.rb @@ -41,7 +41,7 @@ module Gitlab private - attr_reader :json_data, :report, :validate + attr_reader :json_data, :report, :validate, :project def valid? return true unless validate @@ -157,13 +157,7 @@ module Gitlab signature_value: value ) - if signature.valid? - signature - else - e = SecurityReportParserError.new("Vulnerability tracking signature is not valid: #{signature}") - Gitlab::ErrorTracking.track_exception(e) - nil - end + signature if signature.valid? end.compact end diff --git a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb index 627a1f58715..ab5203252a2 100644 --- a/lib/gitlab/ci/parsers/security/validators/schema_validator.rb +++ b/lib/gitlab/ci/parsers/security/validators/schema_validator.rb @@ -7,14 +7,14 @@ module Gitlab module Validators class SchemaValidator SUPPORTED_VERSIONS = { - cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2], - secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2] + cluster_image_scanning: %w[14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + container_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + coverage_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + dast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + api_fuzzing: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + dependency_scanning: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + sast: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4], + secret_detection: %w[14.0.0 14.0.1 14.0.2 14.0.3 14.0.4 14.0.5 14.0.6 14.1.0 14.1.1 14.1.2 14.1.3 15.0.0 15.0.1 15.0.2 15.0.4] }.freeze VERSIONS_TO_REMOVE_IN_16_0 = [].freeze diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json new file mode 100644 index 00000000000..3a859ca8bcf --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/cluster-image-scanning-report-format.json @@ -0,0 +1,984 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/cluster-image-scanning-report-format.json", + "title": "Report format for GitLab Cluster Image Scanning", + "description": "This schema provides the the report format for Cluster Image Scanning (https://docs.gitlab.com/ee/user/application_security/cluster_image_scanning/).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "cluster_image_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "image", + "kubernetes_resource" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image.", + "examples": [ + "index.docker.io/library/nginx:1.21" + ] + }, + "kubernetes_resource": { + "type": "object", + "description": "The specific Kubernetes resource that was scanned.", + "required": [ + "namespace", + "kind", + "name", + "container_name" + ], + "properties": { + "namespace": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes namespace the resource that had its image scanned.", + "examples": [ + "default", + "staging", + "production" + ] + }, + "kind": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The Kubernetes kind the resource that had its image scanned.", + "examples": [ + "Deployment", + "DaemonSet" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the resource that had its image scanned.", + "examples": [ + "nginx-ingress" + ] + }, + "container_name": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The name of the container that had its image scanned.", + "examples": [ + "nginx" + ] + }, + "agent_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes Agent which performed the scan.", + "examples": [ + "1234" + ] + }, + "cluster_id": { + "type": "string", + "minLength": 1, + "maxLength": 255, + "description": "The GitLab ID of the Kubernetes cluster when using cluster integration.", + "examples": [ + "1234" + ] + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json new file mode 100644 index 00000000000..95f9ce90af7 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/container-scanning-report-format.json @@ -0,0 +1,916 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/container-scanning-report-format.json", + "title": "Report format for GitLab Container Scanning", + "description": "This schema provides the the report format for Container Scanning (https://docs.gitlab.com/ee/user/application_security/container_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "container_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "dependency", + "operating_system", + "image" + ], + "properties": { + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + }, + "operating_system": { + "type": "string", + "minLength": 1, + "description": "The operating system that contains the vulnerable package." + }, + "image": { + "type": "string", + "minLength": 1, + "description": "The analyzed Docker image." + }, + "default_branch_image": { + "type": "string", + "maxLength": 255, + "description": "The name of the image on the default branch." + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json new file mode 100644 index 00000000000..b2f39d6f070 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/coverage-fuzzing-report-format.json @@ -0,0 +1,874 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/coverage-fuzzing-report-format.json", + "title": "Report format for GitLab Fuzz Testing", + "description": "This schema provides the report format for Coverage Guided Fuzz Testing (https://docs.gitlab.com/ee/user/application_security/coverage_fuzzing).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "coverage_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "description": "The location of the error", + "type": "object", + "properties": { + "crash_address": { + "type": "string", + "description": "The relative address in memory were the crash occurred.", + "examples": [ + "0xabababab" + ] + }, + "stacktrace_snippet": { + "type": "string", + "description": "The stack trace recorded during fuzzing resulting the crash.", + "examples": [ + "func_a+0xabcd\nfunc_b+0xabcc" + ] + }, + "crash_state": { + "type": "string", + "description": "Minimised and normalized crash stack-trace (called crash_state).", + "examples": [ + "func_a+0xa\nfunc_b+0xb\nfunc_c+0xc" + ] + }, + "crash_type": { + "type": "string", + "description": "Type of the crash.", + "examples": [ + "Heap-Buffer-overflow", + "Division-by-zero" + ] + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json new file mode 100644 index 00000000000..2b86d7e40c9 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dast-report-format.json @@ -0,0 +1,1279 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dast-report-format.json", + "title": "Report format for GitLab DAST", + "description": "This schema provides the the report format for Dynamic Application Security Testing (https://docs.gitlab.com/ee/user/application_security/dast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanned_resources", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dast", + "api_fuzzing" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "scanned_resources": { + "type": "array", + "description": "The attack surface scanned by DAST.", + "items": { + "type": "object", + "required": [ + "method", + "url", + "type" + ], + "properties": { + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method of the scanned resource.", + "examples": [ + "GET", + "POST", + "HEAD" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the scanned resource.", + "examples": [ + "http://my.site.com/a-page" + ] + }, + "type": { + "type": "string", + "minLength": 1, + "description": "Type of the scanned resource, for DAST, this must be 'url'.", + "examples": [ + "url" + ] + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "evidence": { + "type": "object", + "properties": { + "source": { + "type": "object", + "description": "Source of evidence", + "required": [ + "id", + "name" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique source identifier", + "examples": [ + "assert:LogAnalysis", + "assert:StatusCode" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Source display name", + "examples": [ + "Log Analysis", + "Status Code" + ] + }, + "url": { + "type": "string", + "description": "Link to additional information", + "examples": [ + "https://docs.gitlab.com/ee/development/integrations/secure.html" + ] + } + } + }, + "summary": { + "type": "string", + "description": "Human readable string containing evidence of the vulnerability.", + "examples": [ + "Credit card 4111111111111111 found", + "Server leaked information nginx/1.17.6" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + }, + "supporting_messages": { + "type": "array", + "description": "Array of supporting http messages.", + "items": { + "type": "object", + "description": "A supporting http message.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Message display name.", + "examples": [ + "Unmodified", + "Recorded" + ] + }, + "request": { + "type": "object", + "description": "An HTTP request.", + "required": [ + "headers", + "method", + "url" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "method": { + "type": "string", + "minLength": 1, + "description": "HTTP method used in the request.", + "examples": [ + "GET", + "POST" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "URL of the request.", + "examples": [ + "http://my.site.com/vulnerable-endpoint?show-credit-card" + ] + }, + "body": { + "type": "string", + "description": "Body of the request for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "user=jsmith&first=%27&last=smith" + ] + } + } + }, + "response": { + "type": "object", + "description": "An HTTP response.", + "required": [ + "headers", + "reason_phrase", + "status_code" + ], + "properties": { + "headers": { + "type": "array", + "description": "HTTP headers present on the request.", + "items": { + "type": "object", + "required": [ + "name", + "value" + ], + "properties": { + "name": { + "type": "string", + "minLength": 1, + "description": "Name of the HTTP header.", + "examples": [ + "Accept", + "Content-Length", + "Content-Type" + ] + }, + "value": { + "type": "string", + "description": "Value of the HTTP header.", + "examples": [ + "*/*", + "560", + "application/json; charset=utf-8" + ] + } + } + } + }, + "reason_phrase": { + "type": "string", + "description": "HTTP reason phrase of the response.", + "examples": [ + "OK", + "Internal Server Error" + ] + }, + "status_code": { + "type": "integer", + "description": "HTTP status code of the response.", + "examples": [ + 200, + 500 + ] + }, + "body": { + "type": "string", + "description": "Body of the response for display purposes. Body must be suitable for display (not binary), and truncated to a reasonable size.", + "examples": [ + "{\"user_id\": 2}" + ] + } + } + } + } + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "hostname": { + "type": "string", + "description": "The protocol, domain, and port of the application where the vulnerability was found." + }, + "method": { + "type": "string", + "description": "The HTTP method that was used to request the URL where the vulnerability was found." + }, + "param": { + "type": "string", + "description": "A value provided by a vulnerability rule related to the found vulnerability. Examples include a header value, or a parameter used in a HTTP POST." + }, + "path": { + "type": "string", + "description": "The path of the URL where the vulnerability was found. Typically, this would start with a forward slash." + } + } + }, + "assets": { + "type": "array", + "description": "Array of build assets associated with vulnerability.", + "items": { + "type": "object", + "description": "Describes an asset associated with vulnerability.", + "required": [ + "type", + "name", + "url" + ], + "properties": { + "type": { + "type": "string", + "description": "The type of asset", + "enum": [ + "http_session", + "postman" + ] + }, + "name": { + "type": "string", + "minLength": 1, + "description": "Display name for asset", + "examples": [ + "HTTP Messages", + "Postman Collection" + ] + }, + "url": { + "type": "string", + "minLength": 1, + "description": "Link to asset in build artifacts", + "examples": [ + "https://gitlab.com/gitlab-org/security-products/dast/-/jobs/626397001/artifacts/file//output/zap_session.data" + ] + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json new file mode 100644 index 00000000000..29ba60b895e --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/dependency-scanning-report-format.json @@ -0,0 +1,982 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/dependency-scanning-report-format.json", + "title": "Report format for GitLab Dependency Scanning", + "description": "This schema provides the the report format for Dependency Scanning analyzers (https://docs.gitlab.com/ee/user/application_security/dependency_scanning).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "dependency_files", + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "dependency_scanning" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "required": [ + "file", + "dependency" + ], + "properties": { + "file": { + "type": "string", + "minLength": 1, + "description": "Path to the manifest or lock file where the dependency is declared (such as yarn.lock)." + }, + "dependency": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + }, + "dependency_files": { + "type": "array", + "description": "List of dependency files identified in the project.", + "items": { + "type": "object", + "required": [ + "path", + "package_manager", + "dependencies" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "package_manager": { + "type": "string", + "minLength": 1 + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "description": "Describes the dependency of a project where the vulnerability is located.", + "required": [ + "package", + "version" + ], + "properties": { + "package": { + "type": "object", + "description": "Provides information on the package where the vulnerability is located.", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the package where the vulnerability is located." + } + } + }, + "version": { + "type": "string", + "description": "Version of the vulnerable package." + }, + "iid": { + "description": "ID that identifies the dependency in the scope of a dependency file.", + "type": "number" + }, + "direct": { + "type": "boolean", + "description": "Tells whether this is a direct, top-level dependency of the scanned project." + }, + "dependency_path": { + "type": "array", + "description": "Ancestors of the dependency, starting from a direct project dependency, and ending with an immediate parent of the dependency. The dependency itself is excluded from the path. Direct dependencies have no path.", + "items": { + "type": "object", + "required": [ + "iid" + ], + "properties": { + "iid": { + "type": "number", + "description": "ID that is unique in the scope of a parent object, and specific to the resource type." + } + } + } + } + } + } + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json new file mode 100644 index 00000000000..238003f8eb2 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/sast-report-format.json @@ -0,0 +1,869 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/sast-report-format.json", + "title": "Report format for GitLab SAST", + "description": "This schema provides the report format for Static Application Security Testing analyzers (https://docs.gitlab.com/ee/user/application_security/sast).", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "sast" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "type": "object", + "description": "Identifies the vulnerability's location.", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability." + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located." + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located." + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json new file mode 100644 index 00000000000..5cc55ea6409 --- /dev/null +++ b/lib/gitlab/ci/parsers/security/validators/schemas/15.0.4/secret-detection-report-format.json @@ -0,0 +1,893 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://gitlab.com/gitlab-org/security-products/security-report-schemas/-/raw/master/dist/secret-detection-report-format.json", + "title": "Report format for GitLab Secret Detection", + "description": "This schema provides the the report format for the Secret Detection analyzer (https://docs.gitlab.com/ee/user/application_security/secret_detection)", + "definitions": { + "detail_type": { + "oneOf": [ + { + "$ref": "#/definitions/named_list" + }, + { + "$ref": "#/definitions/list" + }, + { + "$ref": "#/definitions/table" + }, + { + "$ref": "#/definitions/text" + }, + { + "$ref": "#/definitions/url" + }, + { + "$ref": "#/definitions/code" + }, + { + "$ref": "#/definitions/value" + }, + { + "$ref": "#/definitions/diff" + }, + { + "$ref": "#/definitions/markdown" + }, + { + "$ref": "#/definitions/commit" + }, + { + "$ref": "#/definitions/file_location" + }, + { + "$ref": "#/definitions/module_location" + } + ] + }, + "text_value": { + "type": "string" + }, + "named_field": { + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "$ref": "#/definitions/text_value", + "type": "string", + "minLength": 1 + }, + "description": { + "$ref": "#/definitions/text_value" + } + } + }, + "named_list": { + "type": "object", + "description": "An object with named and typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "named-list" + }, + "items": { + "type": "object", + "patternProperties": { + "^.*$": { + "allOf": [ + { + "$ref": "#/definitions/named_field" + }, + { + "$ref": "#/definitions/detail_type" + } + ] + } + } + } + } + }, + "list": { + "type": "object", + "description": "A list of typed fields", + "required": [ + "type", + "items" + ], + "properties": { + "type": { + "const": "list" + }, + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + }, + "table": { + "type": "object", + "description": "A table of typed fields", + "required": [ + "type", + "rows" + ], + "properties": { + "type": { + "const": "table" + }, + "header": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + }, + "rows": { + "type": "array", + "items": { + "type": "array", + "items": { + "$ref": "#/definitions/detail_type" + } + } + } + } + }, + "text": { + "type": "object", + "description": "Raw text", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "text" + }, + "value": { + "$ref": "#/definitions/text_value" + } + } + }, + "url": { + "type": "object", + "description": "A single URL", + "required": [ + "type", + "href" + ], + "properties": { + "type": { + "const": "url" + }, + "text": { + "$ref": "#/definitions/text_value" + }, + "href": { + "type": "string", + "minLength": 1, + "examples": [ + "http://mysite.com" + ] + } + } + }, + "code": { + "type": "object", + "description": "A codeblock", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "code" + }, + "value": { + "type": "string" + }, + "lang": { + "type": "string", + "description": "A programming language" + } + } + }, + "value": { + "type": "object", + "description": "A field that can store a range of types of value", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "value" + }, + "value": { + "type": [ + "number", + "string", + "boolean" + ] + } + } + }, + "diff": { + "type": "object", + "description": "A diff", + "required": [ + "type", + "before", + "after" + ], + "properties": { + "type": { + "const": "diff" + }, + "before": { + "type": "string" + }, + "after": { + "type": "string" + } + } + }, + "markdown": { + "type": "object", + "description": "GitLab flavoured markdown, see https://docs.gitlab.com/ee/user/markdown.html", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "markdown" + }, + "value": { + "$ref": "#/definitions/text_value", + "examples": [ + "Here is markdown `inline code` #1 [test](gitlab.com)\n\n" + ] + } + } + }, + "commit": { + "type": "object", + "description": "A commit/tag/branch within the GitLab project", + "required": [ + "type", + "value" + ], + "properties": { + "type": { + "const": "commit" + }, + "value": { + "type": "string", + "description": "The commit SHA", + "minLength": 1 + } + } + }, + "file_location": { + "type": "object", + "description": "A location within a file in the project", + "required": [ + "type", + "file_name", + "line_start" + ], + "properties": { + "type": { + "const": "file-location" + }, + "file_name": { + "type": "string", + "minLength": 1 + }, + "line_start": { + "type": "integer" + }, + "line_end": { + "type": "integer" + } + } + }, + "module_location": { + "type": "object", + "description": "A location within a binary module of the form module+relative_offset", + "required": [ + "type", + "module_name", + "offset" + ], + "properties": { + "type": { + "const": "module-location" + }, + "module_name": { + "type": "string", + "minLength": 1, + "examples": [ + "compiled_binary" + ] + }, + "offset": { + "type": "integer", + "examples": [ + 100 + ] + } + } + } + }, + "self": { + "version": "15.0.4" + }, + "type": "object", + "required": [ + "scan", + "version", + "vulnerabilities" + ], + "additionalProperties": true, + "properties": { + "scan": { + "type": "object", + "required": [ + "analyzer", + "end_time", + "scanner", + "start_time", + "status", + "type" + ], + "properties": { + "end_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan finished.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-01-28T03:26:02" + ] + }, + "messages": { + "type": "array", + "items": { + "type": "object", + "description": "Communication intended for the initiator of a scan.", + "required": [ + "level", + "value" + ], + "properties": { + "level": { + "type": "string", + "description": "Describes the severity of the communication. Use info to communicate normal scan behaviour; warn to communicate a potentially recoverable problem, or a partial error; fatal to communicate an issue that causes the scan to halt.", + "enum": [ + "info", + "warn", + "fatal" + ], + "examples": [ + "info" + ] + }, + "value": { + "type": "string", + "description": "The message to communicate.", + "minLength": 1, + "examples": [ + "Permission denied, scanning aborted" + ] + } + } + } + }, + "analyzer": { + "type": "object", + "description": "Object defining the analyzer used to perform the scan. Analyzers typically delegate to an underlying scanner to run the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the analyzer.", + "minLength": 1, + "examples": [ + "gitlab-dast" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the analyzer, not required to be unique.", + "minLength": 1, + "examples": [ + "GitLab DAST" + ] + }, + "url": { + "type": "string", + "pattern": "^https?://.+", + "description": "A link to more information about the analyzer.", + "examples": [ + "https://docs.gitlab.com/ee/user/application_security/dast" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the analyzer.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + }, + "version": { + "type": "string", + "description": "The version of the analyzer.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + } + } + }, + "scanner": { + "type": "object", + "description": "Object defining the scanner used to perform the scan.", + "required": [ + "id", + "name", + "version", + "vendor" + ], + "properties": { + "id": { + "type": "string", + "description": "Unique id that identifies the scanner.", + "minLength": 1, + "examples": [ + "my-sast-scanner" + ] + }, + "name": { + "type": "string", + "description": "A human readable value that identifies the scanner, not required to be unique.", + "minLength": 1, + "examples": [ + "My SAST Scanner" + ] + }, + "url": { + "type": "string", + "description": "A link to more information about the scanner.", + "examples": [ + "https://scanner.url" + ] + }, + "version": { + "type": "string", + "description": "The version of the scanner.", + "minLength": 1, + "examples": [ + "1.0.2" + ] + }, + "vendor": { + "description": "The vendor/maintainer of the scanner.", + "type": "object", + "required": [ + "name" + ], + "properties": { + "name": { + "type": "string", + "description": "The name of the vendor.", + "minLength": 1, + "examples": [ + "GitLab" + ] + } + } + } + } + }, + "start_time": { + "type": "string", + "description": "ISO8601 UTC value with format yyyy-mm-ddThh:mm:ss, representing when the scan started.", + "pattern": "^\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}$", + "examples": [ + "2020-02-14T16:01:59" + ] + }, + "status": { + "type": "string", + "description": "Result of the scan.", + "enum": [ + "success", + "failure" + ] + }, + "type": { + "type": "string", + "description": "Type of the scan.", + "enum": [ + "secret_detection" + ] + }, + "primary_identifiers": { + "type": "array", + "description": "An unordered array containing an exhaustive list of primary identifiers for which the analyzer may return results", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + } + } + }, + "schema": { + "type": "string", + "description": "URI pointing to the validating security report schema.", + "pattern": "^https?://.+" + }, + "version": { + "type": "string", + "description": "The version of the schema to which the JSON report conforms.", + "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$" + }, + "vulnerabilities": { + "type": "array", + "description": "Array of vulnerability objects.", + "items": { + "type": "object", + "description": "Describes the vulnerability using GitLab Flavored Markdown", + "required": [ + "id", + "identifiers", + "location" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + }, + "name": { + "type": "string", + "maxLength": 255, + "description": "The name of the vulnerability. This must not include the finding's specific information." + }, + "description": { + "type": "string", + "maxLength": 1048576, + "description": "A long text section describing the vulnerability more fully." + }, + "severity": { + "type": "string", + "description": "How much the vulnerability impacts the software. Possible values are Info, Unknown, Low, Medium, High, or Critical. Note that some analyzers may not report all these possible values.", + "enum": [ + "Info", + "Unknown", + "Low", + "Medium", + "High", + "Critical" + ] + }, + "solution": { + "type": "string", + "maxLength": 7000, + "description": "Explanation of how to fix the vulnerability." + }, + "identifiers": { + "type": "array", + "minItems": 1, + "description": "An ordered array of references that identify a vulnerability on internal or external databases. The first identifier is the Primary Identifier, which has special meaning.", + "items": { + "type": "object", + "required": [ + "type", + "name", + "value" + ], + "properties": { + "type": { + "type": "string", + "description": "for example, cve, cwe, osvdb, usn, or an analyzer-dependent type such as gemnasium).", + "minLength": 1 + }, + "name": { + "type": "string", + "description": "Human-readable name of the identifier.", + "minLength": 1 + }, + "url": { + "type": "string", + "description": "URL of the identifier's documentation.", + "pattern": "^https?://.+" + }, + "value": { + "type": "string", + "description": "Value of the identifier, for matching purpose.", + "minLength": 1 + } + } + } + }, + "links": { + "type": "array", + "description": "An array of references to external documentation or articles that describe the vulnerability.", + "items": { + "type": "object", + "required": [ + "url" + ], + "properties": { + "name": { + "type": "string", + "description": "Name of the vulnerability details link." + }, + "url": { + "type": "string", + "description": "URL of the vulnerability details document.", + "pattern": "^https?://.+" + } + } + } + }, + "details": { + "$ref": "#/definitions/named_list/properties/items" + }, + "tracking": { + "type": "object", + "description": "Describes how this vulnerability should be tracked as the project changes.", + "oneOf": [ + { + "description": "Declares that a series of items should be tracked using source-specific tracking methods.", + "required": [ + "items" + ], + "properties": { + "type": { + "const": "source" + }, + "items": { + "type": "array", + "items": { + "description": "An item that should be tracked using source-specific tracking methods.", + "type": "object", + "required": [ + "signatures" + ], + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located." + }, + "start_line": { + "type": "number", + "description": "The first line of the file that includes the vulnerability." + }, + "end_line": { + "type": "number", + "description": "The last line of the file that includes the vulnerability." + }, + "signatures": { + "type": "array", + "description": "An array of calculated tracking signatures for this tracking item.", + "minItems": 1, + "items": { + "description": "A calculated tracking signature value and metadata.", + "type": "object", + "required": [ + "algorithm", + "value" + ], + "properties": { + "algorithm": { + "type": "string", + "description": "The algorithm used to generate the signature." + }, + "value": { + "type": "string", + "description": "The result of this signature algorithm." + } + } + } + } + } + } + } + } + } + ], + "properties": { + "type": { + "type": "string", + "description": "Each tracking type must declare its own type." + } + } + }, + "flags": { + "description": "Flags that can be attached to vulnerabilities.", + "type": "array", + "items": { + "type": "object", + "description": "Informational flags identified and assigned to a vulnerability.", + "required": [ + "type", + "origin", + "description" + ], + "properties": { + "type": { + "type": "string", + "minLength": 1, + "description": "Result of the scan.", + "enum": [ + "flagged-as-likely-false-positive" + ] + }, + "origin": { + "minLength": 1, + "description": "Tool that issued the flag.", + "type": "string" + }, + "description": { + "minLength": 1, + "description": "What the flag is about.", + "type": "string" + } + } + } + }, + "location": { + "required": [ + "commit" + ], + "type": "object", + "properties": { + "file": { + "type": "string", + "description": "Path to the file where the vulnerability is located" + }, + "commit": { + "type": "object", + "description": "Represents the commit in which the vulnerability was detected", + "required": [ + "sha" + ], + "properties": { + "author": { + "type": "string" + }, + "date": { + "type": "string" + }, + "message": { + "type": "string" + }, + "sha": { + "type": "string", + "minLength": 1 + } + } + }, + "start_line": { + "type": "number", + "description": "The first line of the code affected by the vulnerability" + }, + "end_line": { + "type": "number", + "description": "The last line of the code affected by the vulnerability" + }, + "class": { + "type": "string", + "description": "Provides the name of the class where the vulnerability is located" + }, + "method": { + "type": "string", + "description": "Provides the name of the method where the vulnerability is located" + } + } + }, + "raw_source_code_extract": { + "type": "string", + "description": "Provides an unsanitized excerpt of the affected source code." + } + } + } + }, + "remediations": { + "type": "array", + "description": "An array of objects containing information on available remediations, along with patch diffs to apply.", + "items": { + "type": "object", + "required": [ + "fixes", + "summary", + "diff" + ], + "properties": { + "fixes": { + "type": "array", + "description": "An array of strings that represent references to vulnerabilities fixed by this remediation.", + "items": { + "type": "object", + "required": [ + "id" + ], + "properties": { + "id": { + "type": "string", + "minLength": 1, + "description": "Unique identifier of the vulnerability. This is recommended to be a UUID.", + "examples": [ + "642735a5-1425-428d-8d4e-3c854885a3c9" + ] + } + } + } + }, + "summary": { + "type": "string", + "minLength": 1, + "description": "An overview of how the vulnerabilities were fixed." + }, + "diff": { + "type": "string", + "minLength": 1, + "description": "A base64-encoded remediation code diff, compatible with git apply." + } + } + } + } + } +} diff --git a/lib/gitlab/ci/pipeline/chain/command.rb b/lib/gitlab/ci/pipeline/chain/command.rb index 76d4a05bf30..5ec04b4889e 100644 --- a/lib/gitlab/ci/pipeline/chain/command.rb +++ b/lib/gitlab/ci/pipeline/chain/command.rb @@ -117,7 +117,7 @@ module Gitlab logger.observe(:pipeline_size_count, pipeline.total_size) metrics.pipeline_size_histogram - .observe({ source: pipeline.source.to_s }, pipeline.total_size) + .observe({ source: pipeline.source.to_s, plan: project.actual_plan_name }, pipeline.total_size) end def observe_jobs_count_in_alive_pipelines diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index 3dd9b85d9b2..1b9dd158733 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -16,18 +16,7 @@ module Gitlab private def ensure_environment(build) - return unless build.instance_of?(::Ci::Build) && build.has_environment? - - environment = ::Gitlab::Ci::Pipeline::Seed::Environment - .new(build, merge_request: @command.merge_request) - .to_resource - - if environment.persisted? - build.persisted_environment = environment - build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) - else - build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) - end + ::Environments::CreateForBuildService.new.execute(build, merge_request: @command.merge_request) end end end diff --git a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb index 8b26416edf7..2bb32a316be 100644 --- a/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb +++ b/lib/gitlab/ci/pipeline/chain/limit/active_jobs.rb @@ -21,7 +21,10 @@ module Gitlab class: self.class.name, message: MESSAGE, project_id: project.id, - plan: project.actual_plan_name) + plan: project.actual_plan_name, + project_path: project.path, + jobs_in_alive_pipelines_count: count_jobs_in_alive_pipelines + ) end def break? diff --git a/lib/gitlab/ci/pipeline/chain/populate.rb b/lib/gitlab/ci/pipeline/chain/populate.rb index 4bec8355732..654e24be8e1 100644 --- a/lib/gitlab/ci/pipeline/chain/populate.rb +++ b/lib/gitlab/ci/pipeline/chain/populate.rb @@ -25,8 +25,6 @@ module Gitlab return error('Failed to build the pipeline!') end - set_pipeline_name - raise Populate::PopulateError if pipeline.persisted? end @@ -36,15 +34,6 @@ module Gitlab private - def set_pipeline_name - return if Feature.disabled?(:pipeline_name, pipeline.project) || - @command.yaml_processor_result.workflow_name.blank? - - name = @command.yaml_processor_result.workflow_name - - pipeline.build_pipeline_metadata(project: pipeline.project, title: name) - end - def stage_names # We filter out `.pre/.post` stages, as they alone are not considered # a complete pipeline: diff --git a/lib/gitlab/ci/pipeline/chain/populate_metadata.rb b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb new file mode 100644 index 00000000000..35b907b669c --- /dev/null +++ b/lib/gitlab/ci/pipeline/chain/populate_metadata.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Pipeline + module Chain + class PopulateMetadata < Chain::Base + include Chain::Helpers + + def perform! + set_pipeline_name + return if pipeline.pipeline_metadata.nil? || pipeline.pipeline_metadata.valid? + + message = pipeline.pipeline_metadata.errors.full_messages.join(', ') + error("Failed to build pipeline metadata! #{message}") + end + + def break? + pipeline.pipeline_metadata&.errors&.any? + end + + private + + def set_pipeline_name + return if Feature.disabled?(:pipeline_name, pipeline.project) || + @command.yaml_processor_result.workflow_name.blank? + + name = @command.yaml_processor_result.workflow_name + name = ExpandVariables.expand(name, -> { global_context.variables.sort_and_expand_all }) + + pipeline.build_pipeline_metadata(project: pipeline.project, name: name) + end + + def global_context + Gitlab::Ci::Build::Context::Global.new( + pipeline, yaml_variables: @command.pipeline_seed.root_variables) + end + end + end + end + end +end diff --git a/lib/gitlab/ci/pipeline/seed/deployment.rb b/lib/gitlab/ci/pipeline/seed/deployment.rb deleted file mode 100644 index 69dfd6be8d5..00000000000 --- a/lib/gitlab/ci/pipeline/seed/deployment.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Seed - class Deployment < Seed::Base - attr_reader :job, :environment - - def initialize(job, environment) - @job = job - @environment = environment - end - - def to_resource - return job.deployment if job.deployment - return unless job.starts_environment? - - deployment = ::Deployment.new(attributes) - - # If there is a validation error on environment creation, such as - # the name contains invalid character, the job will fall back to a - # non-environment job. - return unless deployment.valid? && deployment.environment.persisted? - - if cluster = deployment.environment.deployment_platform&.cluster - # double write cluster_id until 12.9: https://gitlab.com/gitlab-org/gitlab/issues/202628 - deployment.cluster_id = cluster.id - deployment.deployment_cluster = ::DeploymentCluster.new( - cluster_id: cluster.id, - kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job) - ) - end - - # Allocate IID for deployments. - # This operation must be outside of transactions of pipeline creations. - deployment.ensure_project_iid! - - deployment - end - - private - - def attributes - { - project: job.project, - environment: environment, - user: job.user, - ref: job.ref, - tag: job.tag, - sha: job.sha, - on_stop: job.on_stop - } - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/environment.rb b/lib/gitlab/ci/pipeline/seed/environment.rb deleted file mode 100644 index 8353bc523bf..00000000000 --- a/lib/gitlab/ci/pipeline/seed/environment.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Ci - module Pipeline - module Seed - class Environment < Seed::Base - attr_reader :job, :merge_request - - delegate :simple_variables, to: :job - - def initialize(job, merge_request: nil) - @job = job - @merge_request = merge_request - end - - def to_resource - environments.safe_find_or_create_by(name: expanded_environment_name) do |environment| - # Initialize the attributes at creation - environment.auto_stop_in = expanded_auto_stop_in - environment.tier = deployment_tier - environment.merge_request = merge_request - end - end - - private - - def environments - job.project.environments - end - - def auto_stop_in - job.environment_auto_stop_in - end - - def deployment_tier - job.environment_tier_from_options - end - - def expanded_environment_name - job.expanded_environment_name - end - - def expanded_auto_stop_in - return unless auto_stop_in - - ExpandVariables.expand(auto_stop_in, -> { simple_variables.sort_and_expand_all }) - end - end - end - end - end -end diff --git a/lib/gitlab/ci/pipeline/seed/pipeline.rb b/lib/gitlab/ci/pipeline/seed/pipeline.rb index e1a15fb8d5b..9e609debeed 100644 --- a/lib/gitlab/ci/pipeline/seed/pipeline.rb +++ b/lib/gitlab/ci/pipeline/seed/pipeline.rb @@ -32,6 +32,10 @@ module Gitlab end end + def root_variables + @context.root_variables + end + private def stage_seeds diff --git a/lib/gitlab/ci/reports/codequality_reports.rb b/lib/gitlab/ci/reports/codequality_reports.rb index 353d359fde8..3196bf3fc6d 100644 --- a/lib/gitlab/ci/reports/codequality_reports.rb +++ b/lib/gitlab/ci/reports/codequality_reports.rb @@ -37,8 +37,6 @@ module Gitlab end.to_h end - private - def valid_degradation?(degradation) JSONSchemer.schema(Pathname.new(CODECLIMATE_SCHEMA_PATH)).valid?(degradation) rescue StandardError => _ diff --git a/lib/gitlab/ci/reports/sbom/component.rb b/lib/gitlab/ci/reports/sbom/component.rb index 198b34451b4..5188304f4ed 100644 --- a/lib/gitlab/ci/reports/sbom/component.rb +++ b/lib/gitlab/ci/reports/sbom/component.rb @@ -7,11 +7,34 @@ module Gitlab class Component attr_reader :component_type, :name, :version - def initialize(type:, name:, version:) + def initialize(type:, name:, purl:, version:) @component_type = type @name = name + @purl = purl @version = version end + + def ingestible? + supported_component_type? && supported_purl_type? + end + + def purl + return unless @purl + + ::Sbom::PackageUrl.parse(@purl) + end + + private + + def supported_component_type? + ::Enums::Sbom.component_types.include?(component_type.to_sym) + end + + def supported_purl_type? + return true unless purl + + ::Enums::Sbom.purl_types.include?(purl.type.to_sym) + end end end end diff --git a/lib/gitlab/ci/reports/sbom/report.rb b/lib/gitlab/ci/reports/sbom/report.rb index 4f84d12f78c..51fa8ce0d2e 100644 --- a/lib/gitlab/ci/reports/sbom/report.rb +++ b/lib/gitlab/ci/reports/sbom/report.rb @@ -12,6 +12,10 @@ module Gitlab @errors = [] end + def valid? + errors.empty? + end + def add_error(error) errors << error end diff --git a/lib/gitlab/ci/reports/security/finding.rb b/lib/gitlab/ci/reports/security/finding.rb index 911a7f5d358..dd9b9cc6d55 100644 --- a/lib/gitlab/ci/reports/security/finding.rb +++ b/lib/gitlab/ci/reports/security/finding.rb @@ -156,6 +156,14 @@ module Gitlab signatures.present? end + def false_positive? + flags.any?(&:false_positive?) + end + + def remediation_byte_offsets + remediations.map(&:byte_offsets).compact + end + def raw_metadata @raw_metadata ||= original_data.to_json end @@ -176,6 +184,10 @@ module Gitlab original_data['location'] end + def assets + original_data['assets'] || [] + end + # Returns either the max priority signature hex # or the location fingerprint def location_fingerprint diff --git a/lib/gitlab/ci/reports/security/flag.rb b/lib/gitlab/ci/reports/security/flag.rb index 8370dd60418..e1fbd4c0eff 100644 --- a/lib/gitlab/ci/reports/security/flag.rb +++ b/lib/gitlab/ci/reports/security/flag.rb @@ -27,6 +27,10 @@ module Gitlab description: description }.compact end + + def false_positive? + flag_type == :false_positive + end end end end diff --git a/lib/gitlab/ci/reports/security/reports.rb b/lib/gitlab/ci/reports/security/reports.rb index b6372349f68..5c08381d5cc 100644 --- a/lib/gitlab/ci/reports/security/reports.rb +++ b/lib/gitlab/ci/reports/security/reports.rb @@ -23,6 +23,10 @@ module Gitlab end def violates_default_policy_against?(target_reports, vulnerabilities_allowed, severity_levels, vulnerability_states, report_types = []) + if Feature.enabled?(:require_approval_on_scan_removal, pipeline.project) && scan_removed?(target_reports) + return true + end + unsafe_findings_count(target_reports, severity_levels, vulnerability_states, report_types) > vulnerabilities_allowed end @@ -36,6 +40,10 @@ module Gitlab new_uuids = unsafe_findings_uuids(severity_levels, report_types) - target_reports&.unsafe_findings_uuids(severity_levels, report_types).to_a new_uuids.count end + + def scan_removed?(target_reports) + (target_reports&.reports&.keys.to_a - reports.keys).any? + end end end end diff --git a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml index 004c2897b60..fb062683397 100644 --- a/lib/gitlab/ci/templates/Bash.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Bash.gitlab-ci.yml @@ -41,3 +41,4 @@ deploy1: stage: deploy script: - echo "Do your deploy here" + environment: production diff --git a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml index 01697f67b89..2474bc569d5 100644 --- a/lib/gitlab/ci/templates/Grails.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Grails.gitlab-ci.yml @@ -26,7 +26,7 @@ variables: before_script: - apt-get update -qq && apt-get install -y -qq unzip - curl -sSL https://get.sdkman.io | bash - - echo sdkman_auto_answer=true > ~/.sdkman/etc/config + - echo sdkman_auto_answer=true >> ~/.sdkman/etc/config - source ~/.sdkman/bin/sdkman-init.sh - sdk install gradle $GRADLE_VERSION < /dev/null - sdk use gradle $GRADLE_VERSION diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml index d1018f1e769..fcf2ac7de7a 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html browser_performance: stage: performance diff --git a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml index bb7e020b159..04b7dacf2dd 100644 --- a/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Browser-Performance-Testing.latest.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html browser_performance: stage: performance diff --git a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml index 071eccbab0d..fc1f4f0cce8 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml index 071eccbab0d..fc1f4f0cce8 100644 --- a/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Build.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_BUILD_IMAGE_VERSION: 'v1.19.0' + AUTO_BUILD_IMAGE_VERSION: 'v1.21.0' build: stage: build diff --git a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml index d994ed70ea9..7a208584c4c 100644 --- a/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/DAST-Default-Branch-Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + DAST_AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .dast-auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${DAST_AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml index 7ad71625436..292b0a0036d 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml index 10c843f60a6..ba03ad6304f 100644 --- a/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Deploy.latest.gitlab-ci.yml @@ -1,5 +1,5 @@ variables: - AUTO_DEPLOY_IMAGE_VERSION: 'v2.39.0' + AUTO_DEPLOY_IMAGE_VERSION: 'v2.42.1' .auto-deploy: image: "${CI_TEMPLATE_REGISTRY_HOST}/gitlab-org/cluster-integration/auto-deploy-image:${AUTO_DEPLOY_IMAGE_VERSION}" diff --git a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml index eea1c397108..936d8751fe1 100644 --- a/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Load-Performance-Testing.gitlab-ci.yml @@ -6,7 +6,7 @@ load_performance: DOCKER_TLS_CERTDIR: "" K6_IMAGE: loadimpact/k6 K6_VERSION: 0.27.0 - K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml index 0513aae00a8..77048037915 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST-IaC.latest.gitlab-ci.yml @@ -38,7 +38,7 @@ kics-iac-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /kics/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml index c0ca821ebff..4600468ef30 100644 --- a/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/SAST.latest.gitlab-ci.yml @@ -200,7 +200,7 @@ nodejs-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /nodejs-scan/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/package.json' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -221,7 +221,7 @@ phpcs-security-audit-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /phpcs-security-audit/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.php' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -242,7 +242,7 @@ pmd-apex-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /pmd-apex/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.cls' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -263,7 +263,7 @@ security-code-scan-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /security-code-scan/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.csproj' - '**/*.vbproj' @@ -287,7 +287,7 @@ semgrep-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /semgrep/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.py' - '**/*.js' @@ -326,7 +326,7 @@ sobelow-sast: when: never - if: $SAST_EXCLUDED_ANALYZERS =~ /sobelow/ when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - 'mix.exs' - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. @@ -351,7 +351,7 @@ spotbugs-sast: when: never - if: $SAST_DISABLED when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. exists: - '**/*.groovy' - '**/*.scala' diff --git a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml index e6eba6f6406..6603ee4268e 100644 --- a/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Jobs/Secret-Detection.latest.gitlab-ci.yml @@ -29,7 +29,7 @@ secret_detection: rules: - if: $SECRET_DETECTION_DISABLED when: never - - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. + - if: $CI_PIPELINE_SOURCE == "merge_request_event" # Add the job to merge request pipelines if there's an open merge request. - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. when: never - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml index 1bd527a6ec0..5863da142f0 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-API-Scan.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml index 701e08ba56d..733ba4e4954 100644 --- a/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-On-Demand-Scan.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml index 5b6af37977e..c75ff2e9ff8 100644 --- a/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml @@ -2,6 +2,9 @@ # https://docs.gitlab.com/ee/development/cicd/templates.html # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Security/DAST-Runner-Validation.gitlab-ci.yml +# NOTE: This template is intended for internal GitLab use only and likely will not work properly +# in any other project. Do not include it in your pipeline configuration. +# For information on how to set up and use DAST, visit https://docs.gitlab.com/ee/user/application_security/dast/ stages: - build diff --git a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml index 4d0259fe678..51bcbd278d5 100644 --- a/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - test - build - deploy + - cleanup fmt: extends: .terraform:fmt diff --git a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml index 019b970bc30..0b6c10293fc 100644 --- a/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform.latest.gitlab-ci.yml @@ -12,6 +12,7 @@ stages: - test - build - deploy + - cleanup fmt: extends: .terraform:fmt diff --git a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml index 9a40a23b276..dd1676f25b6 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.gitlab-ci.yml @@ -13,7 +13,7 @@ image: variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project - TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" diff --git a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml index 4579f31d7ac..9c967d48de1 100644 --- a/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Terraform/Base.latest.gitlab-ci.yml @@ -14,7 +14,7 @@ image: variables: TF_ROOT: ${CI_PROJECT_DIR} # The relative path to the root directory of the Terraform project - TF_STATE_NAME: ${TF_STATE_NAME:-default} # The name of the state file used by the GitLab Managed Terraform state backend + TF_STATE_NAME: default # The name of the state file used by the GitLab Managed Terraform state backend cache: key: "${TF_ROOT}" @@ -27,12 +27,22 @@ cache: - cd "${TF_ROOT}" - gitlab-terraform fmt allow_failure: true + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:validate: &terraform_validate stage: validate script: - cd "${TF_ROOT}" - gitlab-terraform validate + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:build: &terraform_build stage: build @@ -46,6 +56,11 @@ cache: - ${TF_ROOT}/plan.cache reports: terraform: ${TF_ROOT}/plan.json + rules: + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + - if: $CI_OPEN_MERGE_REQUESTS # Don't add it to a *branch* pipeline if it's already in a merge request pipeline. + when: never + - if: $CI_COMMIT_BRANCH # If there's no open merge request, add it to a *branch* pipeline instead. .terraform:deploy: &terraform_deploy stage: deploy diff --git a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml index 8a0913e8f66..47329a602b1 100644 --- a/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/ThemeKit.gitlab-ci.yml @@ -8,6 +8,7 @@ stages: - deploy:production staging: + environment: staging image: python:2 stage: deploy:staging script: @@ -18,6 +19,7 @@ staging: - $CI_DEFAULT_BRANCH == $CI_COMMIT_BRANCH production: + environment: production image: python:2 stage: deploy:production script: diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml index 2349c37c130..c3113ffebf3 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html stages: - build diff --git a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml index 73ab5fcbe44..c9f0c173692 100644 --- a/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Browser-Performance.latest.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/browser_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/browser_performance_testing.html stages: - build diff --git a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml index 53fabcfc721..bf5cfbb519d 100644 --- a/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml @@ -3,7 +3,7 @@ # This specific template is located at: # https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Verify/Load-Performance-Testing.gitlab-ci.yml -# Read more about the feature here: https://docs.gitlab.com/ee/user/project/merge_requests/load_performance_testing.html +# Read more about the feature here: https://docs.gitlab.com/ee/ci/testing/code_quality.html stages: - build @@ -17,7 +17,7 @@ load_performance: variables: K6_IMAGE: loadimpact/k6 K6_VERSION: 0.27.0 - K6_TEST_FILE: github.com/loadimpact/k6/samples/http_get.js + K6_TEST_FILE: raw.githubusercontent.com/grafana/k6/master/samples/http_get.js K6_OPTIONS: '' K6_DOCKER_OPTIONS: '' services: diff --git a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml index 50ce181095e..8dfb6c38b55 100644 --- a/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml +++ b/lib/gitlab/ci/templates/dotNET.gitlab-ci.yml @@ -89,3 +89,4 @@ deploy_job: dependencies: - build_job - test_job + environment: production diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index cf5f04215ad..8db8ea3a720 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -171,16 +171,6 @@ module Gitlab end end - def strong_memoize_with(name, *args) - container = strong_memoize(name) { {} } - - if container.key?(args) - container[args] - else - container[args] = yield - end - end - def release return unless @pipeline.tag? diff --git a/lib/gitlab/ci/variables/collection.rb b/lib/gitlab/ci/variables/collection.rb index b6d6e1a3e5f..e9766061072 100644 --- a/lib/gitlab/ci/variables/collection.rb +++ b/lib/gitlab/ci/variables/collection.rb @@ -72,7 +72,8 @@ module Gitlab Collection.new(@variables.reject(&block)) end - def expand_value(value, keep_undefined: false, expand_file_vars: true, project: nil) + # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. + def expand_value(value, keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) value.gsub(Item::VARIABLES_REGEXP) do match = Regexp.last_match # it is either a valid variable definition or a ($$ / %%) full_match = match[0] @@ -86,19 +87,26 @@ module Gitlab variable = self[variable_name] if variable # VARIABLE_NAME is an existing variable - next variable.value unless variable.file? - - # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 - if project - # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` - # when the variables are sent to Runner. - Gitlab::AppJsonLogger.info( - event: 'file_variable_is_referenced_in_another_variable', - project_id: project.id - ) + if variable.file? + # Will be cleaned up with https://gitlab.com/gitlab-org/gitlab/-/issues/378266 + if project + # We only log if `project` exists to make sure it is called from `Ci::BuildRunnerPresenter` + # when the variables are sent to Runner. + Gitlab::AppJsonLogger.info(event: 'file_variable_is_referenced_in_another_variable', + project_id: project.id, + variable: variable_name) + end + + expand_file_refs ? variable.value : full_match + elsif variable.raw? + # With `full_match`, we defer the expansion of raw variables to the runner. If we expand them here, + # the runner will not know the expanded value is a raw variable and it tries to expand it again. + # Discussion: https://gitlab.com/gitlab-org/gitlab/-/issues/353991#note_1103274951 + expand_raw_refs ? variable.value : full_match + else + variable.value end - expand_file_vars ? variable.value : full_match elsif keep_undefined full_match # we do not touch the variable definition else @@ -107,7 +115,8 @@ module Gitlab end end - def sort_and_expand_all(keep_undefined: false, expand_file_vars: true, project: nil) + # `expand_raw_refs` will be deleted with the FF `ci_raw_variables_in_yaml_config`. + def sort_and_expand_all(keep_undefined: false, expand_file_refs: true, expand_raw_refs: true, project: nil) sorted = Sort.new(self) return self.class.new(self, sorted.errors) unless sorted.valid? @@ -122,7 +131,8 @@ module Gitlab # expand variables as they are added variable = item.to_runner_variable variable[:value] = new_collection.expand_value(variable[:value], keep_undefined: keep_undefined, - expand_file_vars: expand_file_vars, + expand_file_refs: expand_file_refs, + expand_raw_refs: expand_raw_refs, project: project) new_collection.append(variable) end diff --git a/lib/gitlab/ci/variables/collection/item.rb b/lib/gitlab/ci/variables/collection/item.rb index ea2aa8f2db8..0fcf11121fa 100644 --- a/lib/gitlab/ci/variables/collection/item.rb +++ b/lib/gitlab/ci/variables/collection/item.rb @@ -21,9 +21,10 @@ module Gitlab @variable.fetch(:value) end - def raw + def raw? @variable.fetch(:raw) end + alias_method :raw, :raw? def file? @variable.fetch(:file) @@ -39,7 +40,7 @@ module Gitlab def depends_on strong_memoize(:depends_on) do - next if raw + next if raw? next unless self.class.possible_var_reference?(value) @@ -48,9 +49,8 @@ module Gitlab end ## - # If `file: true` has been provided we expose it, otherwise we - # don't expose `file` attribute at all (stems from what the runner - # expects). + # If `file: true` or `raw: true` has been provided we expose it, otherwise we + # don't expose `file` and `raw` attributes at all (stems from what the runner expects). # def to_runner_variable @variable.reject do |hash_key, hash_value| diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 5c3864362da..ff255543d3b 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -6,12 +6,17 @@ module Gitlab module Ci class YamlProcessor class Result - attr_reader :errors, :warnings + attr_reader :errors, :warnings, + :root_variables, :root_variables_with_prefill_data, + :stages, :jobs, + :workflow_rules, :workflow_name def initialize(ci_config: nil, errors: [], warnings: []) @ci_config = ci_config @errors = errors || [] @warnings = warnings || [] + + assign_valid_attributes if valid? end def valid? @@ -32,34 +37,10 @@ module Gitlab end end - def workflow_rules - @workflow_rules ||= @ci_config.workflow_rules - end - - def workflow_name - @workflow_name ||= @ci_config.workflow_name&.strip - end - - def root_variables - @root_variables ||= transform_to_array(@ci_config.variables) - end - - def jobs - @jobs ||= @ci_config.normalized_jobs - end - - def stages - @stages ||= @ci_config.stages - end - def included_templates @included_templates ||= @ci_config.included_templates end - def variables_with_data - @ci_config.variables_with_data - end - def yaml_variables_for(job_name) job = jobs[job_name] @@ -82,6 +63,22 @@ module Gitlab private + def assign_valid_attributes + @root_variables = if YamlProcessor::FeatureFlags.enabled?(:ci_raw_variables_in_yaml_config) + transform_to_array(@ci_config.variables_with_data) + else + transform_to_array(@ci_config.variables) + end + + @root_variables_with_prefill_data = @ci_config.variables_with_prefill_data + + @stages = @ci_config.stages + @jobs = @ci_config.normalized_jobs + + @workflow_rules = @ci_config.workflow_rules + @workflow_name = @ci_config.workflow_name&.strip + end + def stage_builds_attributes(stage) jobs.values .select { |job| job[:stage] == stage } @@ -129,14 +126,10 @@ module Gitlab start_in: job[:start_in], trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, - release: release(job) + release: job[:release] }.compact }.compact end - def release(job) - job[:release] - end - def transform_to_array(variables) ::Gitlab::Ci::Variables::Helpers.transform_to_array(variables) end diff --git a/lib/gitlab/cluster/lifecycle_events.rb b/lib/gitlab/cluster/lifecycle_events.rb index be08ada9d2f..b39d2a02f02 100644 --- a/lib/gitlab/cluster/lifecycle_events.rb +++ b/lib/gitlab/cluster/lifecycle_events.rb @@ -63,6 +63,15 @@ module Gitlab # # Sidekiq/Puma Single: This is called immediately. # + # - on_worker_stop (on worker process): + # + # Puma Cluster: Called in the worker process + # exactly once after it stops processing requests + # but before it shuts down. + # + # Sidekiq: Called after the scheduler shuts down but + # before the worker finishes ongoing jobs. + # # Blocks will be executed in the order in which they are registered. # class LifecycleEvents @@ -113,6 +122,10 @@ module Gitlab end end + def on_worker_stop(&block) + (@worker_stop_hooks ||= []) << block + end + # # Lifecycle integration methods (called from puma.rb, etc.) # @@ -137,6 +150,10 @@ module Gitlab call(:master_restart_hooks, @master_restart_hooks) end + def do_worker_stop + call(:worker_stop_hooks, @worker_stop_hooks) + end + # DEPRECATED alias_method :do_master_restart, :do_before_master_restart diff --git a/lib/gitlab/cluster/puma_worker_killer_initializer.rb b/lib/gitlab/cluster/puma_worker_killer_initializer.rb index 5908de68687..957faf797b5 100644 --- a/lib/gitlab/cluster/puma_worker_killer_initializer.rb +++ b/lib/gitlab/cluster/puma_worker_killer_initializer.rb @@ -9,6 +9,10 @@ module Gitlab puma_master_max_memory_mb: 950, additional_puma_dev_max_memory_mb: 200) + # We are replacing PWK with Watchdog by using backward compatible RssMemoryLimit monitor by default. + # https://gitlab.com/groups/gitlab-org/-/epics/9119 + return if Gitlab::Utils.to_boolean(ENV.fetch('GITLAB_MEMORY_WATCHDOG_ENABLED', true)) + require 'puma_worker_killer' PumaWorkerKiller.config do |config| diff --git a/lib/gitlab/config_checker/external_database_checker.rb b/lib/gitlab/config_checker/external_database_checker.rb index 64950fb4eef..ff20833b5be 100644 --- a/lib/gitlab/config_checker/external_database_checker.rb +++ b/lib/gitlab/config_checker/external_database_checker.rb @@ -9,19 +9,23 @@ module Gitlab '<a href="https://docs.gitlab.com/ee/install/requirements.html#database">database requirements</a>' def check - unsupported_database = Gitlab::Database + unsupported_databases = Gitlab::Database .database_base_models - .map { |_, model| Gitlab::Database::Reflection.new(model) } - .reject(&:postgresql_minimum_supported_version?) + .each_with_object({}) do |(database_name, base_model), databases| + database = Gitlab::Database::Reflection.new(base_model) - unsupported_database.map do |database| + databases[database_name] = database unless database.postgresql_minimum_supported_version? + end + + unsupported_databases.map do |database_name, database| { type: 'warning', - message: _('You are using PostgreSQL %{pg_version_current}, but PostgreSQL ' \ - '%{pg_version_minimum} is required for this version of GitLab. ' \ + message: _('Database \'%{database_name}\' is using PostgreSQL %{pg_version_current}, ' \ + 'but PostgreSQL %{pg_version_minimum} is required for this version of GitLab. ' \ 'Please upgrade your environment to a supported PostgreSQL version, ' \ 'see %{pg_requirements_url} for details.') % \ { + database_name: database_name, pg_version_current: database.version, pg_version_minimum: Gitlab::Database::MINIMUM_POSTGRES_VERSION, pg_requirements_url: PG_REQUIREMENTS_LINK diff --git a/lib/gitlab/container_repository/tags/cache.rb b/lib/gitlab/container_repository/tags/cache.rb index 47a6e67a5a1..f9de16f002f 100644 --- a/lib/gitlab/container_repository/tags/cache.rb +++ b/lib/gitlab/container_repository/tags/cache.rb @@ -18,7 +18,7 @@ module Gitlab keys = tags.map(&method(:cache_key)) cached_tags_count = 0 - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| tags.zip(redis.mget(keys)).each do |tag, created_at| next unless created_at @@ -45,7 +45,7 @@ module Gitlab now = Time.zone.now - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| # we use a pipeline instead of a MSET because each tag has # a specific ttl redis.pipelined do |pipeline| @@ -66,6 +66,10 @@ module Gitlab def cache_key(tag) "container_repository:{#{@container_repository.id}}:tag:#{tag.name}:created_at" end + + def with_redis(&block) + ::Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/content_security_policy/config_loader.rb b/lib/gitlab/content_security_policy/config_loader.rb index f1faade250e..29e8e631fb7 100644 --- a/lib/gitlab/content_security_policy/config_loader.rb +++ b/lib/gitlab/content_security_policy/config_loader.rb @@ -24,7 +24,7 @@ module Gitlab 'frame_src' => ContentSecurityPolicy::Directives.frame_src, 'img_src' => "'self' data: blob: http: https:", 'manifest_src' => "'self'", - 'media_src' => "'self' data:", + 'media_src' => "'self' data: http: https:", 'script_src' => ContentSecurityPolicy::Directives.script_src, 'style_src' => "'self' 'unsafe-inline'", 'worker_src' => "#{Gitlab::Utils.append_path(Gitlab.config.gitlab.url, 'assets/')} blob: data:", diff --git a/lib/gitlab/data_builder/build.rb b/lib/gitlab/data_builder/build.rb index 4640f85bb0a..8eda871770b 100644 --- a/lib/gitlab/data_builder/build.rb +++ b/lib/gitlab/data_builder/build.rb @@ -12,7 +12,7 @@ module Gitlab author_url = build_author_url(build.commit, commit) - { + data = { object_kind: 'build', ref: build.ref, @@ -68,6 +68,10 @@ module Gitlab environment: build_environment(build) } + + data[:retries_count] = build.retries_count if Feature.enabled?(:job_webhook_retries_count, project) + + data end private @@ -91,7 +95,7 @@ module Gitlab end def build_environment(build) - return unless build.has_environment? + return unless build.has_environment_keyword? { name: build.expanded_environment_name, diff --git a/lib/gitlab/data_builder/pipeline.rb b/lib/gitlab/data_builder/pipeline.rb index a75c7c539ae..939eaa377aa 100644 --- a/lib/gitlab/data_builder/pipeline.rb +++ b/lib/gitlab/data_builder/pipeline.rb @@ -105,6 +105,7 @@ module Gitlab target_project_id: merge_request.target_project_id, state: merge_request.state, merge_status: merge_request.public_merge_status, + detailed_merge_status: detailed_merge_status(merge_request), url: Gitlab::UrlBuilder.build(merge_request) } end @@ -146,7 +147,7 @@ module Gitlab end def environment_hook_attrs(build) - return unless build.has_environment? + return unless build.has_environment_keyword? { name: build.expanded_environment_name, @@ -154,6 +155,10 @@ module Gitlab deployment_tier: build.persisted_environment.try(:tier) } end + + def detailed_merge_status(merge_request) + ::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: merge_request).execute.to_s + end end end end diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index dd84127459d..04cf056199c 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -56,7 +56,7 @@ module Gitlab # Note that we use ActiveRecord::Base here and not ApplicationRecord. # This is deliberate, as we also use these classes to apply load # balancing to, and the load balancer must be enabled for _all_ models - # that inher from ActiveRecord::Base; not just our own models that + # that inherit from ActiveRecord::Base; not just our own models that # inherit from ApplicationRecord. main: ::ActiveRecord::Base, ci: ::Ci::ApplicationRecord.connection_class? ? ::Ci::ApplicationRecord : nil @@ -217,13 +217,13 @@ module Gitlab Rails.application.config.paths['db'].each do |db_path| path = Rails.root.join(db_path, 'post_migrate').to_s - unless Rails.application.config.paths['db/migrate'].include? path - Rails.application.config.paths['db/migrate'] << path + next if Rails.application.config.paths['db/migrate'].include? path - # Rails memoizes migrations at certain points where it won't read the above - # path just yet. As such we must also update the following list of paths. - ActiveRecord::Migrator.migrations_paths << path - end + Rails.application.config.paths['db/migrate'] << path + + # Rails memoizes migrations at certain points where it won't read the above + # path just yet. As such we must also update the following list of paths. + ActiveRecord::Migrator.migrations_paths << path end end diff --git a/lib/gitlab/database/background_migration/batched_job.rb b/lib/gitlab/database/background_migration/batched_job.rb index 81898a59da7..6b7ff308c7e 100644 --- a/lib/gitlab/database/background_migration/batched_job.rb +++ b/lib/gitlab/database/background_migration/batched_job.rb @@ -14,7 +14,8 @@ module Gitlab MAX_ATTEMPTS = 3 STUCK_JOBS_TIMEOUT = 1.hour.freeze TIMEOUT_EXCEPTIONS = [ActiveRecord::StatementTimeout, ActiveRecord::ConnectionTimeoutError, - ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout].freeze + ActiveRecord::AdapterTimeout, ActiveRecord::LockWaitTimeout, + ActiveRecord::QueryCanceled].freeze belongs_to :batched_migration, foreign_key: :batched_background_migration_id has_many :batched_job_transition_logs, foreign_key: :batched_background_migration_job_id @@ -112,7 +113,10 @@ module Gitlab end def can_split?(exception) - attempts >= MAX_ATTEMPTS && TIMEOUT_EXCEPTIONS.include?(exception&.class) && batch_size > sub_batch_size && batch_size > 1 + attempts >= MAX_ATTEMPTS && + exception&.class&.in?(TIMEOUT_EXCEPTIONS) && + batch_size > sub_batch_size && + batch_size > 1 end def split_and_retry! diff --git a/lib/gitlab/database/background_migration/batched_migration.rb b/lib/gitlab/database/background_migration/batched_migration.rb index 92cafd1d00e..61a660ad14c 100644 --- a/lib/gitlab/database/background_migration/batched_migration.rb +++ b/lib/gitlab/database/background_migration/batched_migration.rb @@ -94,8 +94,21 @@ module Gitlab end def self.active_migration(connection:) + active_migrations_distinct_on_table(connection: connection, limit: 1).first + end + + def self.find_executable(id, connection:) for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) - .executable.queue_order.first + .executable.find_by_id(id) + end + + def self.active_migrations_distinct_on_table(connection:, limit:) + distinct_on_table = select('DISTINCT ON (table_name) id') + .for_gitlab_schema(Gitlab::Database.gitlab_schemas_for_connection(connection)) + .executable + .order(table_name: :asc, id: :asc) + + where(id: distinct_on_table).queue_order.limit(limit) end def self.successful_rows_counts(migrations) diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index c4a9cf8b80f..bf6ebb21f7d 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -40,6 +40,7 @@ atlassian_identities: :gitlab_main audit_events_external_audit_event_destinations: :gitlab_main audit_events: :gitlab_main audit_events_streaming_headers: :gitlab_main +audit_events_streaming_event_type_filters: :gitlab_main authentication_events: :gitlab_main award_emoji: :gitlab_main aws_roles: :gitlab_main @@ -167,6 +168,7 @@ dast_site_profiles_pipelines: :gitlab_main dast_sites: :gitlab_main dast_site_tokens: :gitlab_main dast_site_validations: :gitlab_main +dependency_proxy_blob_states: :gitlab_main dependency_proxy_blobs: :gitlab_main dependency_proxy_group_settings: :gitlab_main dependency_proxy_image_ttl_group_policies: :gitlab_main @@ -206,7 +208,6 @@ events: :gitlab_main evidences: :gitlab_main experiments: :gitlab_main experiment_subjects: :gitlab_main -experiment_users: :gitlab_main external_approval_rules: :gitlab_main external_approval_rules_protected_branches: :gitlab_main external_pull_requests: :gitlab_ci @@ -342,6 +343,7 @@ namespace_limits: :gitlab_main namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_ci_cd_settings: :gitlab_main +namespace_commit_emails: :gitlab_main namespace_settings: :gitlab_main namespace_details: :gitlab_main namespaces: :gitlab_main @@ -363,6 +365,7 @@ operations_scopes: :gitlab_main operations_strategies: :gitlab_main operations_strategies_user_lists: :gitlab_main operations_user_lists: :gitlab_main +p_ci_builds_metadata: :gitlab_ci packages_build_infos: :gitlab_main packages_cleanup_policies: :gitlab_main packages_composer_cache_files: :gitlab_main @@ -451,6 +454,7 @@ projects: :gitlab_main projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main +project_wiki_repositories: :gitlab_main project_wiki_repository_states: :gitlab_main prometheus_alert_events: :gitlab_main prometheus_alerts: :gitlab_main diff --git a/lib/gitlab/database/load_balancing/configuration.rb b/lib/gitlab/database/load_balancing/configuration.rb index 59b08fac7e9..50472bd5780 100644 --- a/lib/gitlab/database/load_balancing/configuration.rb +++ b/lib/gitlab/database/load_balancing/configuration.rb @@ -57,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil } end diff --git a/lib/gitlab/database/load_balancing/load_balancer.rb b/lib/gitlab/database/load_balancing/load_balancer.rb index 0881025b425..cb3a378ad64 100644 --- a/lib/gitlab/database/load_balancing/load_balancer.rb +++ b/lib/gitlab/database/load_balancing/load_balancer.rb @@ -119,6 +119,13 @@ module Gitlab connection = pool.connection transaction_open = connection.transaction_open? + if attempt && attempt > 1 + ::Gitlab::Database::LoadBalancing::Logger.warn( + event: :read_write_retry, + message: 'A read_write block was retried because of connection error' + ) + end + yield connection rescue StandardError => e # No leaking will happen on the final attempt. Leaks are caused by subsequent retries diff --git a/lib/gitlab/database/load_balancing/service_discovery.rb b/lib/gitlab/database/load_balancing/service_discovery.rb index dfd4892371c..52a9e8798d4 100644 --- a/lib/gitlab/database/load_balancing/service_discovery.rb +++ b/lib/gitlab/database/load_balancing/service_discovery.rb @@ -48,6 +48,7 @@ module Gitlab # forcefully disconnected. # use_tcp - Use TCP instaed of UDP to look up resources # load_balancer - The load balancer instance to use + # rubocop:disable Metrics/ParameterLists def initialize( load_balancer, nameserver:, @@ -56,7 +57,8 @@ module Gitlab record_type: 'A', interval: 60, disconnect_timeout: 120, - use_tcp: false + use_tcp: false, + max_replica_pools: nil ) @nameserver = nameserver @port = port @@ -66,7 +68,9 @@ module Gitlab @disconnect_timeout = disconnect_timeout @use_tcp = use_tcp @load_balancer = load_balancer + @max_replica_pools = max_replica_pools end + # rubocop:enable Metrics/ParameterLists def start Thread.new do @@ -170,6 +174,8 @@ module Gitlab addresses_from_srv_record(response) end + addresses = sampler.sample(addresses) + raise EmptyDnsResponse if addresses.empty? # Addresses are sorted so we can directly compare the old and new @@ -221,6 +227,11 @@ module Gitlab def addresses_from_a_record(resources) resources.map { |r| Address.new(r.address.to_s) } end + + def sampler + @sampler ||= ::Gitlab::Database::LoadBalancing::ServiceDiscovery::Sampler + .new(max_replica_pools: @max_replica_pools) + end end end end diff --git a/lib/gitlab/database/load_balancing/service_discovery/sampler.rb b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb new file mode 100644 index 00000000000..71870214156 --- /dev/null +++ b/lib/gitlab/database/load_balancing/service_discovery/sampler.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module LoadBalancing + class ServiceDiscovery + class Sampler + def initialize(max_replica_pools:, seed: Random.new_seed) + # seed must be set once and consistent + # for every invocation of #sample on + # the same instance of Sampler + @seed = seed + @max_replica_pools = max_replica_pools + end + + def sample(addresses) + return addresses if @max_replica_pools.nil? || addresses.count <= @max_replica_pools + + ::Gitlab::Database::LoadBalancing::Logger.info( + event: :host_list_limit_exceeded, + message: "Host list length exceeds max_replica_pools so random hosts will be chosen.", + max_replica_pools: @max_replica_pools, + total_host_list_length: addresses.count, + randomization_seed: @seed + ) + + # First sort them in case the ordering from DNS server changes + # then randomly order all addresses using consistent seed so + # this process always gives the same set for this instance of + # Sampler + addresses = addresses.sort + addresses = addresses.shuffle(random: Random.new(@seed)) + + # Group by hostname so that we can sample evenly across hosts + addresses_by_host = addresses.group_by(&:hostname) + + selected_addresses = [] + while selected_addresses.count < @max_replica_pools + # Loop over all hostnames grabbing one address at a time to + # evenly distribute across all hostnames + addresses_by_host.each do |host, addresses| + next if addresses.empty? + + selected_addresses << addresses.pop + + break unless selected_addresses.count < @max_replica_pools + end + end + + selected_addresses + end + end + end + end + end +end diff --git a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb index 3180289ec69..737852d5ccb 100644 --- a/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb +++ b/lib/gitlab/database/load_balancing/sidekiq_server_middleware.rb @@ -4,7 +4,7 @@ module Gitlab module Database module LoadBalancing class SidekiqServerMiddleware - JobReplicaNotUpToDate = Class.new(StandardError) + JobReplicaNotUpToDate = Class.new(::Gitlab::SidekiqMiddleware::RetryError) MINIMUM_DELAY_INTERVAL_SECONDS = 0.8 diff --git a/lib/gitlab/database/lock_writes_manager.rb b/lib/gitlab/database/lock_writes_manager.rb index fe75cd763b4..2594ee04b35 100644 --- a/lib/gitlab/database/lock_writes_manager.rb +++ b/lib/gitlab/database/lock_writes_manager.rb @@ -5,6 +5,11 @@ module Gitlab class LockWritesManager TRIGGER_FUNCTION_NAME = 'gitlab_schema_prevent_write' + # Triggers to block INSERT / UPDATE / DELETE + # Triggers on TRUNCATE are not added to the information_schema.triggers + # See https://www.postgresql.org/message-id/16934.1568989957%40sss.pgh.pa.us + EXPECTED_TRIGGER_RECORD_COUNT = 3 + def initialize(table_name:, connection:, database_name:, logger: nil, dry_run: false) @table_name = table_name @connection = connection @@ -20,7 +25,7 @@ module Gitlab AND trigger_name = '#{write_trigger_name(table_name)}' SQL - connection.select_value(query) == 3 + connection.select_value(query) == EXPECTED_TRIGGER_RECORD_COUNT end def lock_writes diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index df40e3b3868..16416dd2507 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -6,6 +6,10 @@ module Gitlab include Migrations::ReestablishedConnectionStack include Migrations::BackgroundMigrationHelpers include Migrations::BatchedBackgroundMigrationHelpers + include Migrations::LockRetriesHelpers + include Migrations::TimeoutHelpers + include Migrations::ConstraintsHelpers + include Migrations::ExtensionHelpers include DynamicModelHelpers include RenameTableHelpers include AsyncIndexes::MigrationHelpers @@ -22,8 +26,6 @@ module Gitlab super(table_name, connection: connection, **kwargs) end - # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS - MAX_IDENTIFIER_NAME_LENGTH = 63 DEFAULT_TIMESTAMP_COLUMNS = %i[created_at updated_at].freeze # Adds `created_at` and `updated_at` columns with timezone information. @@ -146,6 +148,12 @@ module Gitlab 'in the body of your migration class' end + if !options.delete(:allow_partition) && partition?(table_name) + raise ArgumentError, 'add_concurrent_index can not be used on a partitioned ' \ + 'table. Please use add_concurrent_partitioned_index on the partitioned table ' \ + 'as we need to create indexes on each partition and an index on the parent table' + end + options = options.merge({ algorithm: :concurrently }) if index_exists?(table_name, column_name, **options) @@ -202,6 +210,12 @@ module Gitlab 'in the body of your migration class' end + if partition?(table_name) + raise ArgumentError, 'remove_concurrent_index can not be used on a partitioned ' \ + 'table. Please use remove_concurrent_partitioned_index_by_name on the partitioned table ' \ + 'as we need to remove the index on the parent table' + end + options = options.merge({ algorithm: :concurrently }) unless index_exists?(table_name, column_name, **options) @@ -231,6 +245,12 @@ module Gitlab 'in the body of your migration class' end + if partition?(table_name) + raise ArgumentError, 'remove_concurrent_index_by_name can not be used on a partitioned ' \ + 'table. Please use remove_concurrent_partitioned_index_by_name on the partitioned table ' \ + 'as we need to remove the index on the parent table' + end + index_name = index_name[:name] if index_name.is_a?(Hash) raise 'remove_concurrent_index_by_name must get an index name as the second argument' if index_name.blank? @@ -360,97 +380,6 @@ module Gitlab "#{prefix}#{hashed_identifier}" end - # Long-running migrations may take more than the timeout allowed by - # the database. Disable the session's statement timeout to ensure - # migrations don't get killed prematurely. - # - # There are two possible ways to disable the statement timeout: - # - # - Per transaction (this is the preferred and default mode) - # - Per connection (requires a cleanup after the execution) - # - # When using a per connection disable statement, code must be inside - # a block so we can automatically execute `RESET statement_timeout` after block finishes - # otherwise the statement will still be disabled until connection is dropped - # or `RESET statement_timeout` is executed - def disable_statement_timeout - if block_given? - if statement_timeout_disabled? - # Don't do anything if the statement_timeout is already disabled - # Allows for nested calls of disable_statement_timeout without - # resetting the timeout too early (before the outer call ends) - yield - else - begin - execute('SET statement_timeout TO 0') - - yield - ensure - execute('RESET statement_timeout') - end - end - else - unless transaction_open? - raise <<~ERROR - Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block. - If you don't want to use a transaction wrap your code in a block call: - - disable_statement_timeout { # code that requires disabled statement here } - - This will make sure statement_timeout is disabled before and reset after the block execution is finished. - ERROR - end - - execute('SET LOCAL statement_timeout TO 0') - end - end - - # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. - # The timings can be controlled via the +timing_configuration+ parameter. - # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. - # - # Note this helper uses subtransactions when run inside an already open transaction. - # - # ==== Examples - # # Invoking without parameters - # with_lock_retries do - # drop_table :my_table - # end - # - # # Invoking with custom +timing_configuration+ - # t = [ - # [1.second, 1.second], - # [2.seconds, 2.seconds] - # ] - # - # with_lock_retries(timing_configuration: t) do - # drop_table :my_table # this will be retried twice - # end - # - # # Disabling the retries using an environment variable - # > export DISABLE_LOCK_RETRIES=true - # - # with_lock_retries do - # drop_table :my_table # one invocation, it will not retry at all - # end - # - # ==== Parameters - # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the block, sleep time before the next iteration, defaults to `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` - # * +logger+ - [Gitlab::JsonLogger] - # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` - def with_lock_retries(*args, **kwargs, &block) - raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) - merged_args = { - connection: connection, - klass: self.class, - logger: Gitlab::BackgroundMigration::Logger, - allow_savepoints: true - }.merge(kwargs) - - Gitlab::Database::WithLockRetries.new(**merged_args) - .run(raise_on_exhaustion: raise_on_exhaustion, &block) - end - def true_value Database.true_value end @@ -796,6 +725,10 @@ module Gitlab install_rename_triggers(table, old, new) end + def convert_to_type_column(column, from_type, to_type) + "#{column}_convert_#{from_type}_to_#{to_type}" + end + def convert_to_bigint_column(column) "#{column}_convert_to_bigint" end @@ -826,7 +759,22 @@ module Gitlab # columns - The name, or array of names, of the column(s) that we want to convert to bigint. # primary_key - The name of the primary key column (most often :id) def initialize_conversion_of_integer_to_bigint(table, columns, primary_key: :id) - create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :bigint) + mappings = Array(columns).map do |c| + { + c => { + from_type: :int, + to_type: :bigint, + default_value: 0 + } + } + end.reduce(&:merge) + + create_temporary_columns_and_triggers( + table, + mappings, + primary_key: primary_key, + old_bigint_column_naming: true + ) end # Reverts `initialize_conversion_of_integer_to_bigint` @@ -849,9 +797,23 @@ module Gitlab # table - The name of the database table containing the columns # columns - The name, or array of names, of the column(s) that we have converted to bigint. # primary_key - The name of the primary key column (most often :id) - def restore_conversion_of_integer_to_bigint(table, columns, primary_key: :id) - create_temporary_columns_and_triggers(table, columns, primary_key: primary_key, data_type: :int) + mappings = Array(columns).map do |c| + { + c => { + from_type: :bigint, + to_type: :int, + default_value: 0 + } + } + end.reduce(&:merge) + + create_temporary_columns_and_triggers( + table, + mappings, + primary_key: primary_key, + old_bigint_column_naming: true + ) end # Backfills the new columns used in an integer-to-bigint conversion using background migrations. @@ -947,43 +909,6 @@ module Gitlab execute("DELETE FROM batched_background_migrations WHERE #{conditions}") end - def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) - Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! - - Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information - migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( - Gitlab::Database.gitlab_schemas_for_connection(connection), - job_class_name, table_name, column_name, job_arguments - ) - - configuration = { - job_class_name: job_class_name, - table_name: table_name, - column_name: column_name, - job_arguments: job_arguments - } - - return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? - - return if migration.finished? - - finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize - - unless migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload - raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ - "but it is '#{migration.status_name}':" \ - "\t#{configuration}" \ - "\n\n" \ - "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ - "\n\n" \ - "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ - "\n\n" \ - "For more information, check the documentation" \ - "\n\n" \ - "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" - end - end - # Returns an Array containing the indexes for the given column def indexes_for(table, column) column = column.to_s @@ -1102,6 +1027,24 @@ module Gitlab rescue ArgumentError end + # Remove any instances of deprecated job classes lingering in queues. + # + # rubocop:disable Cop/SidekiqApiUsage + def sidekiq_remove_jobs(job_klass:) + Sidekiq::Queue.new(job_klass.queue).each do |job| + job.delete if job.klass == job_klass.to_s + end + + Sidekiq::RetrySet.new.each do |retri| + retri.delete if retri.klass == job_klass.to_s + end + + Sidekiq::ScheduledSet.new.each do |scheduled| + scheduled.delete if scheduled.klass == job_klass.to_s + end + end + # rubocop:enable Cop/SidekiqApiUsage + def sidekiq_queue_migrate(queue_from, to:) while sidekiq_queue_length(queue_from) > 0 Sidekiq.redis do |conn| @@ -1194,320 +1137,6 @@ into similar problems in the future (e.g. when new tables are created). execute(sql) end - # Returns the name for a check constraint - # - # type: - # - Any value, as long as it is unique - # - Constraint names are unique per table in Postgres, and, additionally, - # we can have multiple check constraints over a column - # So we use the (table, column, type) triplet as a unique name - # - e.g. we use 'max_length' when adding checks for text limits - # or 'not_null' when adding a NOT NULL constraint - # - def check_constraint_name(table, column, type) - identifier = "#{table}_#{column}_check_#{type}" - # Check concurrent_foreign_key_name() for info on why we use a hash - hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) - - "check_#{hashed_identifier}" - end - - def check_constraint_exists?(table, constraint_name) - # Constraint names are unique per table in Postgres, not per schema - # Two tables can have constraints with the same name, so we filter by - # the table name in addition to using the constraint_name - check_sql = <<~SQL - SELECT COUNT(*) - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - WHERE con.contype = 'c' - AND con.conname = #{connection.quote(constraint_name)} - AND nsp.nspname = #{connection.quote(current_schema)} - AND rel.relname = #{connection.quote(table)} - SQL - - connection.select_value(check_sql) > 0 - end - - # Adds a check constraint to a table - # - # This method is the generic helper for adding any check constraint - # More specialized helpers may use it (e.g. add_text_limit or add_not_null) - # - # This method only requires minimal locking: - # - The constraint is added using NOT VALID - # This allows us to add the check constraint without validating it - # - The check will be enforced for new data (inserts) coming in - # - If `validate: true` the constraint is also validated - # Otherwise, validate_check_constraint() can be used at a later stage - # - Check comments on add_concurrent_foreign_key for more info - # - # table - The table the constraint will be added to - # check - The check clause to add - # e.g. 'char_length(name) <= 5' or 'store IS NOT NULL' - # constraint_name - The name of the check constraint (otherwise auto-generated) - # Should be unique per table (not per column) - # validate - Whether to validate the constraint in this call - # - def add_check_constraint(table, check, constraint_name, validate: true) - # Transactions would result in ALTER TABLE locks being held for the - # duration of the transaction, defeating the purpose of this method. - validate_not_in_transaction!(:add_check_constraint) - - validate_check_constraint_name!(constraint_name) - - if check_constraint_exists?(table, constraint_name) - warning_message = <<~MESSAGE - Check constraint was not created because it exists already - (this may be due to an aborted migration or similar) - table: #{table}, check: #{check}, constraint name: #{constraint_name} - MESSAGE - - Gitlab::AppLogger.warn warning_message - else - # Only add the constraint without validating it - # Even though it is fast, ADD CONSTRAINT requires an EXCLUSIVE lock - # Use with_lock_retries to make sure that this operation - # will not timeout on tables accessed by many processes - with_lock_retries do - execute <<-EOF.strip_heredoc - ALTER TABLE #{table} - ADD CONSTRAINT #{constraint_name} - CHECK ( #{check} ) - NOT VALID; - EOF - end - end - - if validate - validate_check_constraint(table, constraint_name) - end - end - - def validate_check_constraint(table, constraint_name) - validate_check_constraint_name!(constraint_name) - - unless check_constraint_exists?(table, constraint_name) - raise missing_schema_object_message(table, "check constraint", constraint_name) - end - - disable_statement_timeout do - # VALIDATE CONSTRAINT only requires a SHARE UPDATE EXCLUSIVE LOCK - # It only conflicts with other validations and creating indexes - execute("ALTER TABLE #{table} VALIDATE CONSTRAINT #{constraint_name};") - end - end - - def remove_check_constraint(table, constraint_name) - # This is technically not necessary, but aligned with add_check_constraint - # and allows us to continue use with_lock_retries here - validate_not_in_transaction!(:remove_check_constraint) - - validate_check_constraint_name!(constraint_name) - - # DROP CONSTRAINT requires an EXCLUSIVE lock - # Use with_lock_retries to make sure that this will not timeout - with_lock_retries do - execute <<-EOF.strip_heredoc - ALTER TABLE #{table} - DROP CONSTRAINT IF EXISTS #{constraint_name} - EOF - end - end - - # Copies all check constraints for the old column to the new column. - # - # table - The table containing the columns. - # old - The old column. - # new - The new column. - # schema - The schema the table is defined for - # If it is not provided, then the current_schema is used - def copy_check_constraints(table, old, new, schema: nil) - if transaction_open? - raise 'copy_check_constraints can not be run inside a transaction' - end - - unless column_exists?(table, old) - raise "Column #{old} does not exist on #{table}" - end - - unless column_exists?(table, new) - raise "Column #{new} does not exist on #{table}" - end - - table_with_schema = schema.present? ? "#{schema}.#{table}" : table - - check_constraints_for(table, old, schema: schema).each do |check_c| - validate = !(check_c["constraint_def"].end_with? "NOT VALID") - - # Normalize: - # - Old constraint definitions: - # '(char_length(entity_path) <= 5500)' - # - Definitionss from pg_get_constraintdef(oid): - # 'CHECK ((char_length(entity_path) <= 5500))' - # - Definitions from pg_get_constraintdef(oid, pretty_bool): - # 'CHECK (char_length(entity_path) <= 5500)' - # - Not valid constraints: 'CHECK (...) NOT VALID' - # to a single format that we can use: - # '(char_length(entity_path) <= 5500)' - check_definition = check_c["constraint_def"] - .sub(/^\s*(CHECK)?\s*\({0,2}/, '(') - .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')') - - constraint_name = begin - if check_definition == "(#{old} IS NOT NULL)" - not_null_constraint_name(table_with_schema, new) - elsif check_definition.start_with? "(char_length(#{old}) <=" - text_limit_name(table_with_schema, new) - else - check_constraint_name(table_with_schema, new, 'copy_check_constraint') - end - end - - add_check_constraint( - table_with_schema, - check_definition.gsub(old.to_s, new.to_s), - constraint_name, - validate: validate - ) - end - end - - # Migration Helpers for adding limit to text columns - def add_text_limit(table, column, limit, constraint_name: nil, validate: true) - add_check_constraint( - table, - "char_length(#{column}) <= #{limit}", - text_limit_name(table, column, name: constraint_name), - validate: validate - ) - end - - def validate_text_limit(table, column, constraint_name: nil) - validate_check_constraint(table, text_limit_name(table, column, name: constraint_name)) - end - - def remove_text_limit(table, column, constraint_name: nil) - remove_check_constraint(table, text_limit_name(table, column, name: constraint_name)) - end - - def check_text_limit_exists?(table, column, constraint_name: nil) - check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name)) - end - - # Migration Helpers for managing not null constraints - def add_not_null_constraint(table, column, constraint_name: nil, validate: true) - if column_is_nullable?(table, column) - add_check_constraint( - table, - "#{column} IS NOT NULL", - not_null_constraint_name(table, column, name: constraint_name), - validate: validate - ) - else - warning_message = <<~MESSAGE - NOT NULL check constraint was not created: - column #{table}.#{column} is already defined as `NOT NULL` - MESSAGE - - Gitlab::AppLogger.warn warning_message - end - end - - def validate_not_null_constraint(table, column, constraint_name: nil) - validate_check_constraint( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def remove_not_null_constraint(table, column, constraint_name: nil) - remove_check_constraint( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def check_not_null_constraint_exists?(table, column, constraint_name: nil) - check_constraint_exists?( - table, - not_null_constraint_name(table, column, name: constraint_name) - ) - end - - def create_extension(extension) - execute('CREATE EXTENSION IF NOT EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but - the database user is not allowed to install the extension. - - You can either install the extension manually using a database superuser: - - CREATE EXTENSION IF NOT EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be installed through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - - def drop_extension(extension) - execute('DROP EXTENSION IF EXISTS %s' % extension) - rescue ActiveRecord::StatementInvalid => e - dbname = ApplicationRecord.database.database_name - user = ApplicationRecord.database.username - - warn(<<~MSG) if e.to_s =~ /permission denied/ - This migration attempts to drop the PostgreSQL extension '#{extension}' - installed in database '#{dbname}', but the database user is not allowed - to drop the extension. - - You can either drop the extension manually using a database superuser: - - DROP EXTENSION IF EXISTS #{extension} - - Or, you can solve this by logging in to the GitLab - database (#{dbname}) using a superuser and running: - - ALTER #{user} WITH SUPERUSER - - This query will grant the user superuser permissions, ensuring any database extensions - can be dropped through migrations. - - For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. - MSG - - raise - end - - def rename_constraint(table_name, old_name, new_name) - execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} - RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} - SQL - end - - def drop_constraint(table_name, constraint_name, cascade: false) - execute <<~SQL - ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} - SQL - end - def add_primary_key_using_index(table_name, pk_name, index_to_use) execute <<~SQL ALTER TABLE #{quote_table_name(table_name)} ADD CONSTRAINT #{quote_table_name(pk_name)} PRIMARY KEY USING INDEX #{quote_table_name(index_to_use)} @@ -1536,17 +1165,20 @@ into similar problems in the future (e.g. when new tables are created). SQL end - private + # rubocop:disable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity + def create_temporary_columns_and_triggers(table, mappings, primary_key: :id, old_bigint_column_naming: false) + raise ArgumentError, "No mappings for column conversion provided" if mappings.blank? - def multiple_columns(columns, separator: ', ') - Array.wrap(columns).join(separator) - end + unless mappings.values.all? { |values| mapping_has_required_columns?(values) } + raise ArgumentError, "Some mappings don't have required keys provided" + end - def cascade_statement(cascade) - cascade ? 'CASCADE' : '' - end + neutral_values_for_type = { + int: 0, + bigint: 0, + uuid: '00000000-0000-0000-0000-000000000000' + } - def create_temporary_columns_and_triggers(table, columns, primary_key: :id, data_type: :bigint) unless table_exists?(table) raise "Table #{table} does not exist" end @@ -1555,7 +1187,7 @@ into similar problems in the future (e.g. when new tables are created). raise "Column #{primary_key} does not exist on #{table}" end - columns = Array.wrap(columns) + columns = mappings.keys columns.each do |column| next if column_exists?(table, column) @@ -1564,67 +1196,88 @@ into similar problems in the future (e.g. when new tables are created). check_trigger_permissions!(table) - conversions = columns.to_h { |column| [column, convert_to_bigint_column(column)] } + if old_bigint_column_naming + mappings.each do |column, params| + params.merge!( + temporary_column_name: convert_to_bigint_column(column) + ) + end + else + mappings.each do |column, params| + params.merge!( + temporary_column_name: convert_to_type_column(column, params[:from_type], params[:to_type]) + ) + end + end with_lock_retries do - conversions.each do |(source_column, temporary_name)| - column = column_for(table, source_column) + mappings.each do |(column_name, params)| + column = column_for(table, column_name) + temporary_name = params[:temporary_column_name] + data_type = params[:to_type] + default_value = params[:default_value] if (column.name.to_s == primary_key.to_s) || !column.null # If the column to be converted is either a PK or is defined as NOT NULL, # set it to `NOT NULL DEFAULT 0` and we'll copy paste the correct values bellow # That way, we skip the expensive validation step required to add # a NOT NULL constraint at the end of the process - add_column(table, temporary_name, data_type, default: column.default || 0, null: false) + add_column( + table, + temporary_name, + data_type, + default: column.default || default_value || neutral_values_for_type.fetch(data_type), + null: false + ) else - add_column(table, temporary_name, data_type, default: column.default) + add_column( + table, + temporary_name, + data_type, + default: column.default + ) end end - install_rename_triggers(table, conversions.keys, conversions.values) + old_column_names = mappings.keys + temporary_column_names = mappings.values.map { |v| v[:temporary_column_name] } + install_rename_triggers(table, old_column_names, temporary_column_names) end end + # rubocop:enable Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity - def validate_check_constraint_name!(constraint_name) - if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH - raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + def partition?(table_name) + if view_exists?(:postgres_partitions) + Gitlab::Database::PostgresPartition.partition_exists?(table_name) + else + Gitlab::Database::PostgresPartition.legacy_partition_exists?(table_name) end end - # Returns an ActiveRecord::Result containing the check constraints - # defined for the given column. - # - # If the schema is not provided, then the current_schema is used - def check_constraints_for(table, column, schema: nil) - check_sql = <<~SQL - SELECT - ccu.table_schema as schema_name, - ccu.table_name as table_name, - ccu.column_name as column_name, - con.conname as constraint_name, - pg_get_constraintdef(con.oid) as constraint_def - FROM pg_catalog.pg_constraint con - INNER JOIN pg_catalog.pg_class rel - ON rel.oid = con.conrelid - INNER JOIN pg_catalog.pg_namespace nsp - ON nsp.oid = con.connamespace - INNER JOIN information_schema.constraint_column_usage ccu - ON con.conname = ccu.constraint_name - AND nsp.nspname = ccu.constraint_schema - AND rel.relname = ccu.table_name - WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)} - AND rel.relname = #{connection.quote(table)} - AND ccu.column_name = #{connection.quote(column)} - AND con.contype = 'c' - ORDER BY constraint_name - SQL + private + + def multiple_columns(columns, separator: ', ') + Array.wrap(columns).join(separator) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end - connection.exec_query(check_sql) + def validate_check_constraint_name!(constraint_name) + if constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH + raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + end end - def statement_timeout_disabled? - # This is a string of the form "100ms" or "0" when disabled - connection.select_value('SHOW statement_timeout') == "0" + # mappings => {} where keys are column names and values are hashes with the following keys: + # from_type - from which type we're migrating + # to_type - to which type we're migrating + # default_value - custom default value, if not provided will be taken from neutral_values_for_type + def mapping_has_required_columns?(mapping) + %i[from_type to_type].map do |required_key| + mapping.has_key?(required_key) + end.all? end def column_is_nullable?(table, column) @@ -1640,14 +1293,6 @@ into similar problems in the future (e.g. when new tables are created). connection.select_value(check_sql) == 'YES' end - def text_limit_name(table, column, name: nil) - name.presence || check_constraint_name(table, column, 'max_length') - end - - def not_null_constraint_name(table, column, name: nil) - name.presence || check_constraint_name(table, column, 'not_null') - end - def missing_schema_object_message(table, type, name) <<~MESSAGE Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. @@ -1717,17 +1362,6 @@ into similar problems in the future (e.g. when new tables are created). Must end with `_at`} MESSAGE end - - def validate_not_in_transaction!(method_name, modifier = nil) - return unless transaction_open? - - raise <<~ERROR - #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. - - You can disable transactions by calling `disable_ddl_transaction!` in the body of - your migration class - ERROR - end end end end diff --git a/lib/gitlab/database/migration_helpers/v2.rb b/lib/gitlab/database/migration_helpers/v2.rb index dd426962033..b5b8b58681c 100644 --- a/lib/gitlab/database/migration_helpers/v2.rb +++ b/lib/gitlab/database/migration_helpers/v2.rb @@ -205,8 +205,8 @@ module Gitlab raise "Column #{old_column} does not exist on #{table}" end - if column.default - raise "#{calling_operation} does not currently support columns with default values" + if column.default_function + raise "#{calling_operation} does not currently support columns with default functions" end unless column_exists?(table, batch_column_name) @@ -269,17 +269,20 @@ module Gitlab def create_insert_trigger(trigger_name, quoted_table, quoted_old_column, quoted_new_column) function_name = function_name_for_trigger(trigger_name) + column = columns(quoted_table.delete('"').to_sym).find { |column| column.name == quoted_old_column.delete('"') } + quoted_default_value = connection.quote(column.default) + execute(<<~SQL) CREATE OR REPLACE FUNCTION #{function_name}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN - IF NEW.#{quoted_old_column} IS NULL AND NEW.#{quoted_new_column} IS NOT NULL THEN + IF NEW.#{quoted_old_column} IS NOT DISTINCT FROM #{quoted_default_value} AND NEW.#{quoted_new_column} IS DISTINCT FROM #{quoted_default_value} THEN NEW.#{quoted_old_column} = NEW.#{quoted_new_column}; END IF; - IF NEW.#{quoted_new_column} IS NULL AND NEW.#{quoted_old_column} IS NOT NULL THEN + IF NEW.#{quoted_new_column} IS NOT DISTINCT FROM #{quoted_default_value} AND NEW.#{quoted_old_column} IS DISTINCT FROM #{quoted_default_value} THEN NEW.#{quoted_new_column} = NEW.#{quoted_old_column}; END IF; diff --git a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb index 363fd0598f9..e958ce2aba4 100644 --- a/lib/gitlab/database/migrations/batched_background_migration_helpers.rb +++ b/lib/gitlab/database/migrations/batched_background_migration_helpers.rb @@ -196,6 +196,43 @@ module Gitlab :gitlab_main end end + + def ensure_batched_background_migration_is_finished(job_class_name:, table_name:, column_name:, job_arguments:, finalize: true) + Gitlab::Database::QueryAnalyzers::RestrictAllowedSchemas.require_dml_mode! + + Gitlab::Database::BackgroundMigration::BatchedMigration.reset_column_information + migration = Gitlab::Database::BackgroundMigration::BatchedMigration.find_for_configuration( + Gitlab::Database.gitlab_schemas_for_connection(connection), + job_class_name, table_name, column_name, job_arguments + ) + + configuration = { + job_class_name: job_class_name, + table_name: table_name, + column_name: column_name, + job_arguments: job_arguments + } + + return Gitlab::AppLogger.warn "Could not find batched background migration for the given configuration: #{configuration}" if migration.nil? + + return if migration.finished? + + finalize_batched_background_migration(job_class_name: job_class_name, table_name: table_name, column_name: column_name, job_arguments: job_arguments) if finalize + + return if migration.reload.finished? # rubocop:disable Cop/ActiveRecordAssociationReload + + raise "Expected batched background migration for the given configuration to be marked as 'finished', " \ + "but it is '#{migration.status_name}':" \ + "\t#{configuration}" \ + "\n\n" \ + "Finalize it manually by running the following command in a `bash` or `sh` shell:" \ + "\n\n" \ + "\tsudo gitlab-rake gitlab:background_migrations:finalize[#{job_class_name},#{table_name},#{column_name},'#{job_arguments.to_json.gsub(',', '\,')}']" \ + "\n\n" \ + "For more information, check the documentation" \ + "\n\n" \ + "\thttps://docs.gitlab.com/ee/user/admin_area/monitoring/background_migrations.html#database-migrations-failing-because-of-batched-background-migration-not-finished" + end end end end diff --git a/lib/gitlab/database/migrations/constraints_helpers.rb b/lib/gitlab/database/migrations/constraints_helpers.rb new file mode 100644 index 00000000000..7b849e3137a --- /dev/null +++ b/lib/gitlab/database/migrations/constraints_helpers.rb @@ -0,0 +1,337 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ConstraintsHelpers + include LockRetriesHelpers + include TimeoutHelpers + + # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS + MAX_IDENTIFIER_NAME_LENGTH = 63 + + # Returns the name for a check constraint + # + # type: + # - Any value, as long as it is unique + # - Constraint names are unique per table in Postgres, and, additionally, + # we can have multiple check constraints over a column + # So we use the (table, column, type) triplet as a unique name + # - e.g. we use 'max_length' when adding checks for text limits + # or 'not_null' when adding a NOT NULL constraint + # + def check_constraint_name(table, column, type) + identifier = "#{table}_#{column}_check_#{type}" + # Check concurrent_foreign_key_name() for info on why we use a hash + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + + "check_#{hashed_identifier}" + end + + def check_constraint_exists?(table, constraint_name) + # Constraint names are unique per table in Postgres, not per schema + # Two tables can have constraints with the same name, so we filter by + # the table name in addition to using the constraint_name + + check_sql = <<~SQL + SELECT COUNT(*) + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + WHERE con.contype = 'c' + AND con.conname = #{connection.quote(constraint_name)} + AND nsp.nspname = #{connection.quote(current_schema)} + AND rel.relname = #{connection.quote(table)} + SQL + + connection.select_value(check_sql) > 0 + end + + # Adds a check constraint to a table + # + # This method is the generic helper for adding any check constraint + # More specialized helpers may use it (e.g. add_text_limit or add_not_null) + # + # This method only requires minimal locking: + # - The constraint is added using NOT VALID + # This allows us to add the check constraint without validating it + # - The check will be enforced for new data (inserts) coming in + # - If `validate: true` the constraint is also validated + # Otherwise, validate_check_constraint() can be used at a later stage + # - Check comments on add_concurrent_foreign_key for more info + # + # table - The table the constraint will be added to + # check - The check clause to add + # e.g. 'char_length(name) <= 5' or 'store IS NOT NULL' + # constraint_name - The name of the check constraint (otherwise auto-generated) + # Should be unique per table (not per column) + # validate - Whether to validate the constraint in this call + # + def add_check_constraint(table, check, constraint_name, validate: true) + # Transactions would result in ALTER TABLE locks being held for the + # duration of the transaction, defeating the purpose of this method. + validate_not_in_transaction!(:add_check_constraint) + + validate_check_constraint_name!(constraint_name) + + if check_constraint_exists?(table, constraint_name) + warning_message = <<~MESSAGE + Check constraint was not created because it exists already + (this may be due to an aborted migration or similar) + table: #{table}, check: #{check}, constraint name: #{constraint_name} + MESSAGE + + Gitlab::AppLogger.warn warning_message + else + # Only add the constraint without validating it + # Even though it is fast, ADD CONSTRAINT requires an EXCLUSIVE lock + # Use with_lock_retries to make sure that this operation + # will not timeout on tables accessed by many processes + with_lock_retries do + execute <<~SQL + ALTER TABLE #{table} + ADD CONSTRAINT #{constraint_name} + CHECK ( #{check} ) + NOT VALID; + SQL + end + end + + validate_check_constraint(table, constraint_name) if validate + end + + def validate_check_constraint(table, constraint_name) + validate_check_constraint_name!(constraint_name) + + unless check_constraint_exists?(table, constraint_name) + raise missing_schema_object_message(table, "check constraint", constraint_name) + end + + disable_statement_timeout do + # VALIDATE CONSTRAINT only requires a SHARE UPDATE EXCLUSIVE LOCK + # It only conflicts with other validations and creating indexes + execute("ALTER TABLE #{table} VALIDATE CONSTRAINT #{constraint_name};") + end + end + + def remove_check_constraint(table, constraint_name) + # This is technically not necessary, but aligned with add_check_constraint + # and allows us to continue use with_lock_retries here + validate_not_in_transaction!(:remove_check_constraint) + + validate_check_constraint_name!(constraint_name) + + # DROP CONSTRAINT requires an EXCLUSIVE lock + # Use with_lock_retries to make sure that this will not timeout + with_lock_retries do + execute <<-SQL + ALTER TABLE #{table} + DROP CONSTRAINT IF EXISTS #{constraint_name} + SQL + end + end + + # Copies all check constraints for the old column to the new column. + # + # table - The table containing the columns. + # old - The old column. + # new - The new column. + # schema - The schema the table is defined for + # If it is not provided, then the current_schema is used + def copy_check_constraints(table, old, new, schema: nil) + raise 'copy_check_constraints can not be run inside a transaction' if transaction_open? + + raise "Column #{old} does not exist on #{table}" unless column_exists?(table, old) + + raise "Column #{new} does not exist on #{table}" unless column_exists?(table, new) + + table_with_schema = schema.present? ? "#{schema}.#{table}" : table + + check_constraints_for(table, old, schema: schema).each do |check_c| + validate = !(check_c["constraint_def"].end_with? "NOT VALID") + + # Normalize: + # - Old constraint definitions: + # '(char_length(entity_path) <= 5500)' + # - Definitionss from pg_get_constraintdef(oid): + # 'CHECK ((char_length(entity_path) <= 5500))' + # - Definitions from pg_get_constraintdef(oid, pretty_bool): + # 'CHECK (char_length(entity_path) <= 5500)' + # - Not valid constraints: 'CHECK (...) NOT VALID' + # to a single format that we can use: + # '(char_length(entity_path) <= 5500)' + check_definition = check_c["constraint_def"] + .sub(/^\s*(CHECK)?\s*\({0,2}/, '(') + .sub(/\){0,2}\s*(NOT VALID)?\s*$/, ')') + + constraint_name = if check_definition == "(#{old} IS NOT NULL)" + not_null_constraint_name(table_with_schema, new) + elsif check_definition.start_with? "(char_length(#{old}) <=" + text_limit_name(table_with_schema, new) + else + check_constraint_name(table_with_schema, new, 'copy_check_constraint') + end + + add_check_constraint( + table_with_schema, + check_definition.gsub(old.to_s, new.to_s), + constraint_name, + validate: validate + ) + end + end + + # Migration Helpers for adding limit to text columns + def add_text_limit(table, column, limit, constraint_name: nil, validate: true) + add_check_constraint( + table, + "char_length(#{column}) <= #{limit}", + text_limit_name(table, column, name: constraint_name), + validate: validate + ) + end + + def validate_text_limit(table, column, constraint_name: nil) + validate_check_constraint(table, text_limit_name(table, column, name: constraint_name)) + end + + def remove_text_limit(table, column, constraint_name: nil) + remove_check_constraint(table, text_limit_name(table, column, name: constraint_name)) + end + + def check_text_limit_exists?(table, column, constraint_name: nil) + check_constraint_exists?(table, text_limit_name(table, column, name: constraint_name)) + end + + # Migration Helpers for managing not null constraints + def add_not_null_constraint(table, column, constraint_name: nil, validate: true) + if column_is_nullable?(table, column) + add_check_constraint( + table, + "#{column} IS NOT NULL", + not_null_constraint_name(table, column, name: constraint_name), + validate: validate + ) + else + warning_message = <<~MESSAGE + NOT NULL check constraint was not created: + column #{table}.#{column} is already defined as `NOT NULL` + MESSAGE + + Gitlab::AppLogger.warn warning_message + end + end + + def validate_not_null_constraint(table, column, constraint_name: nil) + validate_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def remove_not_null_constraint(table, column, constraint_name: nil) + remove_check_constraint( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def check_not_null_constraint_exists?(table, column, constraint_name: nil) + check_constraint_exists?( + table, + not_null_constraint_name(table, column, name: constraint_name) + ) + end + + def rename_constraint(table_name, old_name, new_name) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} + RENAME CONSTRAINT #{quote_column_name(old_name)} TO #{quote_column_name(new_name)} + SQL + end + + def drop_constraint(table_name, constraint_name, cascade: false) + execute <<~SQL + ALTER TABLE #{quote_table_name(table_name)} DROP CONSTRAINT #{quote_column_name(constraint_name)} #{cascade_statement(cascade)} + SQL + end + + def validate_check_constraint_name!(constraint_name) + return unless constraint_name.to_s.length > MAX_IDENTIFIER_NAME_LENGTH + + raise "The maximum allowed constraint name is #{MAX_IDENTIFIER_NAME_LENGTH} characters" + end + + def text_limit_name(table, column, name: nil) + name.presence || check_constraint_name(table, column, 'max_length') + end + + private + + def validate_not_in_transaction!(method_name, modifier = nil) + return unless transaction_open? + + raise <<~ERROR + #{["`#{method_name}`", modifier].compact.join(' ')} cannot be run inside a transaction. + + You can disable transactions by calling `disable_ddl_transaction!` in the body of + your migration class + ERROR + end + + # Returns an ActiveRecord::Result containing the check constraints + # defined for the given column. + # + # If the schema is not provided, then the current_schema is used + def check_constraints_for(table, column, schema: nil) + check_sql = <<~SQL + SELECT + ccu.table_schema as schema_name, + ccu.table_name as table_name, + ccu.column_name as column_name, + con.conname as constraint_name, + pg_get_constraintdef(con.oid) as constraint_def + FROM pg_catalog.pg_constraint con + INNER JOIN pg_catalog.pg_class rel + ON rel.oid = con.conrelid + INNER JOIN pg_catalog.pg_namespace nsp + ON nsp.oid = con.connamespace + INNER JOIN information_schema.constraint_column_usage ccu + ON con.conname = ccu.constraint_name + AND nsp.nspname = ccu.constraint_schema + AND rel.relname = ccu.table_name + WHERE nsp.nspname = #{connection.quote(schema.presence || current_schema)} + AND rel.relname = #{connection.quote(table)} + AND ccu.column_name = #{connection.quote(column)} + AND con.contype = 'c' + ORDER BY constraint_name + SQL + + connection.exec_query(check_sql) + end + + def cascade_statement(cascade) + cascade ? 'CASCADE' : '' + end + + def not_null_constraint_name(table, column, name: nil) + name.presence || check_constraint_name(table, column, 'not_null') + end + + def missing_schema_object_message(table, type, name) + <<~MESSAGE + Could not find #{type} "#{name}" on table "#{table}" which was referenced during the migration. + This issue could be caused by the database schema straying from the expected state. + + To resolve this issue, please verify: + 1. all previous migrations have completed + 2. the database objects used in this migration match the Rails definition in schema.rb or structure.sql + + MESSAGE + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/extension_helpers.rb b/lib/gitlab/database/migrations/extension_helpers.rb new file mode 100644 index 00000000000..435e9e0d2dc --- /dev/null +++ b/lib/gitlab/database/migrations/extension_helpers.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module ExtensionHelpers + def create_extension(extension) + execute("CREATE EXTENSION IF NOT EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + GitLab requires the PostgreSQL extension '#{extension}' installed in database '#{dbname}', but + the database user is not allowed to install the extension. + + You can either install the extension manually using a database superuser: + + CREATE EXTENSION IF NOT EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be installed through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + + def drop_extension(extension) + execute("DROP EXTENSION IF EXISTS #{extension}") + rescue ActiveRecord::StatementInvalid => e + dbname = ApplicationRecord.database.database_name + user = ApplicationRecord.database.username + + warn(<<~MSG) if e.to_s.include?('permission denied') + This migration attempts to drop the PostgreSQL extension '#{extension}' + installed in database '#{dbname}', but the database user is not allowed + to drop the extension. + + You can either drop the extension manually using a database superuser: + + DROP EXTENSION IF EXISTS #{extension} + + Or, you can solve this by logging in to the GitLab + database (#{dbname}) using a superuser and running: + + ALTER #{user} WITH SUPERUSER + + This query will grant the user superuser permissions, ensuring any database extensions + can be dropped through migrations. + + For more information, refer to https://docs.gitlab.com/ee/install/postgresql_extensions.html. + MSG + + raise + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/lock_retries_helpers.rb b/lib/gitlab/database/migrations/lock_retries_helpers.rb new file mode 100644 index 00000000000..137ef3ab144 --- /dev/null +++ b/lib/gitlab/database/migrations/lock_retries_helpers.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module LockRetriesHelpers + # Executes the block with a retry mechanism that alters the +lock_timeout+ and +sleep_time+ between attempts. + # The timings can be controlled via the +timing_configuration+ parameter. + # If the lock was not acquired within the retry period, a last attempt is made without using +lock_timeout+. + # + # Note this helper uses subtransactions when run inside an already open transaction. + # + # ==== Examples + # # Invoking without parameters + # with_lock_retries do + # drop_table :my_table + # end + # + # # Invoking with custom +timing_configuration+ + # t = [ + # [1.second, 1.second], + # [2.seconds, 2.seconds] + # ] + # + # with_lock_retries(timing_configuration: t) do + # drop_table :my_table # this will be retried twice + # end + # + # # Disabling the retries using an environment variable + # > export DISABLE_LOCK_RETRIES=true + # + # with_lock_retries do + # drop_table :my_table # one invocation, it will not retry at all + # end + # + # ==== Parameters + # * +timing_configuration+ - [[ActiveSupport::Duration, ActiveSupport::Duration], ...] lock timeout for the + # block, sleep time before the next iteration, defaults to + # `Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION` + # * +logger+ - [Gitlab::JsonLogger] + # * +env+ - [Hash] custom environment hash, see the example with `DISABLE_LOCK_RETRIES` + def with_lock_retries(*args, **kwargs, &block) + raise_on_exhaustion = !!kwargs.delete(:raise_on_exhaustion) + merged_args = { + connection: connection, + klass: self.class, + logger: Gitlab::BackgroundMigration::Logger, + allow_savepoints: true + }.merge(kwargs) + + Gitlab::Database::WithLockRetries.new(**merged_args) + .run(raise_on_exhaustion: raise_on_exhaustion, &block) + end + end + end + end +end diff --git a/lib/gitlab/database/migrations/runner.rb b/lib/gitlab/database/migrations/runner.rb index 85dc6051c7c..27b161419b2 100644 --- a/lib/gitlab/database/migrations/runner.rb +++ b/lib/gitlab/database/migrations/runner.rb @@ -7,6 +7,7 @@ module Gitlab BASE_RESULT_DIR = Rails.root.join('tmp', 'migration-testing').freeze METADATA_FILENAME = 'metadata.json' SCHEMA_VERSION = 4 # Version of the output format produced by the runner + POST_MIGRATION_MATCHER = %r{db/post_migrate/}.freeze class << self def up(database:, legacy_mode: false) @@ -116,7 +117,10 @@ module Gitlab verbose_was = ActiveRecord::Migration.verbose ActiveRecord::Migration.verbose = true - sorted_migrations = migrations.sort_by(&:version) + sorted_migrations = migrations.sort_by do |m| + [m.filename.match?(POST_MIGRATION_MATCHER) ? 1 : 0, m.version] + end + sorted_migrations.reverse! if direction == :down instrumentation = Instrumentation.new(result_dir: result_dir) diff --git a/lib/gitlab/database/migrations/timeout_helpers.rb b/lib/gitlab/database/migrations/timeout_helpers.rb new file mode 100644 index 00000000000..423c77452b1 --- /dev/null +++ b/lib/gitlab/database/migrations/timeout_helpers.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Migrations + module TimeoutHelpers + # Long-running migrations may take more than the timeout allowed by + # the database. Disable the session's statement timeout to ensure + # migrations don't get killed prematurely. + # + # There are two possible ways to disable the statement timeout: + # + # - Per transaction (this is the preferred and default mode) + # - Per connection (requires a cleanup after the execution) + # + # When using a per connection disable statement, code must be inside + # a block so we can automatically execute `RESET statement_timeout` after block finishes + # otherwise the statement will still be disabled until connection is dropped + # or `RESET statement_timeout` is executed + def disable_statement_timeout + if block_given? + if statement_timeout_disabled? + # Don't do anything if the statement_timeout is already disabled + # Allows for nested calls of disable_statement_timeout without + # resetting the timeout too early (before the outer call ends) + yield + else + begin + execute('SET statement_timeout TO 0') + + yield + ensure + execute('RESET statement_timeout') + end + end + else + unless transaction_open? + raise <<~ERROR + Cannot call disable_statement_timeout() without a transaction open or outside of a transaction block. + If you don't want to use a transaction wrap your code in a block call: + + disable_statement_timeout { # code that requires disabled statement here } + + This will make sure statement_timeout is disabled before and reset after the block execution is finished. + ERROR + end + + execute('SET LOCAL statement_timeout TO 0') + end + end + + private + + def statement_timeout_disabled? + # This is a string of the form "100ms" or "0" when disabled + connection.select_value('SHOW statement_timeout') == "0" + end + end + end + end +end diff --git a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb index 23a8dc0b44f..58447481e60 100644 --- a/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb +++ b/lib/gitlab/database/partitioning/convert_table_to_first_list_partition.rb @@ -10,13 +10,17 @@ module Gitlab attr_reader :partitioning_column, :table_name, :parent_table_name, :zero_partition_value - def initialize(migration_context:, table_name:, parent_table_name:, partitioning_column:, zero_partition_value:) + def initialize( + migration_context:, table_name:, parent_table_name:, partitioning_column:, + zero_partition_value:, lock_tables: []) + @migration_context = migration_context @connection = migration_context.connection @table_name = table_name @parent_table_name = parent_table_name @partitioning_column = partitioning_column @zero_partition_value = zero_partition_value + @lock_tables = Array.wrap(lock_tables) end def prepare_for_partitioning @@ -35,7 +39,12 @@ module Gitlab create_parent_table attach_foreign_keys_to_parent - migration_context.with_lock_retries(raise_on_exhaustion: true) do + lock_args = { + raise_on_exhaustion: true, + timing_configuration: lock_timing_configuration + } + + migration_context.with_lock_retries(**lock_args) do migration_context.execute(sql_to_convert_table) end end @@ -74,6 +83,7 @@ module Gitlab # but they acquire the same locks so it's much faster to incude them # here. [ + lock_tables_statement, attach_table_to_parent_statement, alter_sequence_statements(old_table: table_name, new_table: parent_table_name), remove_constraint_statement @@ -162,6 +172,16 @@ module Gitlab end end + def lock_tables_statement + return if @lock_tables.empty? + + table_names = @lock_tables.map { |name| quote_table_name(name) }.join(', ') + + <<~SQL + LOCK #{table_names} IN ACCESS EXCLUSIVE MODE + SQL + end + def attach_table_to_parent_statement <<~SQL ALTER TABLE #{quote_table_name(parent_table_name)} @@ -235,6 +255,13 @@ module Gitlab ALTER TABLE #{connection.quote_table_name(table_name)} OWNER TO CURRENT_USER SQL end + + def lock_timing_configuration + iterations = Gitlab::Database::WithLockRetries::DEFAULT_TIMING_CONFIGURATION + aggressive_iterations = Array.new(5) { [10.seconds, 1.minute] } + + iterations + aggressive_iterations + end end end end diff --git a/lib/gitlab/database/partitioning/detached_partition_dropper.rb b/lib/gitlab/database/partitioning/detached_partition_dropper.rb index 5e32ecad4ca..58c0728b614 100644 --- a/lib/gitlab/database/partitioning/detached_partition_dropper.rb +++ b/lib/gitlab/database/partitioning/detached_partition_dropper.rb @@ -7,7 +7,7 @@ module Gitlab Gitlab::AppLogger.info(message: "Checking for previously detached partitions to drop") Postgresql::DetachedPartition.ready_to_drop.find_each do |detached_partition| - if partition_attached?(qualify_partition_name(detached_partition.table_name)) + if partition_attached?(detached_partition.fully_qualified_table_name) unmark_partition(detached_partition) else drop_partition(detached_partition) @@ -41,14 +41,14 @@ module Gitlab # Another process may have already dropped the table and deleted this entry next unless try_lock_detached_partition(detached_partition.id) - drop_detached_partition(detached_partition.table_name) + drop_detached_partition(detached_partition) detached_partition.destroy! end end def remove_foreign_keys(detached_partition) - partition_identifier = qualify_partition_name(detached_partition.table_name) + partition_identifier = detached_partition.fully_qualified_table_name # We want to load all of these into memory at once to get a consistent view to loop over, # since we'll be deleting from this list as we go @@ -65,7 +65,7 @@ module Gitlab # It is important to only drop one foreign key per transaction. # Dropping a foreign key takes an ACCESS EXCLUSIVE lock on both tables participating in the foreign key. - partition_identifier = qualify_partition_name(detached_partition.table_name) + partition_identifier = detached_partition.fully_qualified_table_name with_lock_retries do connection.transaction(requires_new: false) do next unless try_lock_detached_partition(detached_partition.id) @@ -83,16 +83,10 @@ module Gitlab end end - def drop_detached_partition(partition_name) - partition_identifier = qualify_partition_name(partition_name) + def drop_detached_partition(detached_partition) + connection.drop_table(detached_partition.fully_qualified_table_name, if_exists: true) - connection.drop_table(partition_identifier, if_exists: true) - - Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: partition_name) - end - - def qualify_partition_name(table_name) - "#{Gitlab::Database::DYNAMIC_PARTITIONS_SCHEMA}.#{table_name}" + Gitlab::AppLogger.info(message: "Dropped previously detached partition", partition_name: detached_partition.table_name) end def partition_attached?(partition_identifier) diff --git a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb index 15b542cf089..62f33bb56bc 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/index_helpers.rb @@ -7,6 +7,8 @@ module Gitlab include Gitlab::Database::MigrationHelpers include Gitlab::Database::SchemaHelpers + DuplicatedIndexesError = Class.new(StandardError) + ERROR_SCOPE = 'index' # Concurrently creates a new index on a partitioned table. In concept this works similarly to @@ -38,7 +40,7 @@ module Gitlab partitioned_table.postgres_partitions.order(:name).each do |partition| partition_index_name = generated_index_name(partition.identifier, options[:name]) - partition_options = options.merge(name: partition_index_name) + partition_options = options.merge(name: partition_index_name, allow_partition: true) add_concurrent_index(partition.identifier, column_names, partition_options) end @@ -92,6 +94,42 @@ module Gitlab .map { |_, indexes| indexes.map { |index| index['index_name'] } } end + # Retrieves a hash of index names for a given table and schema, by index + # definition. + # + # Example: + # + # indexes_by_definition_for_table('table_name_goes_here') + # + # Returns: + # + # { + # "CREATE _ btree (created_at)" => "index_on_created_at" + # } + def indexes_by_definition_for_table(table_name, schema_name: connection.current_schema) + duplicate_indexes = find_duplicate_indexes(table_name, schema_name: schema_name) + + unless duplicate_indexes.empty? + raise DuplicatedIndexesError, "#{table_name} has duplicate indexes: #{duplicate_indexes}" + end + + find_indexes(table_name, schema_name: schema_name) + .each_with_object({}) { |row, hash| hash[row['index_id']] = row['index_name'] } + end + + # Renames indexes for a given table and schema, mapping by index + # definition, to a hash of new index names. + # + # Example: + # + # index_names = indexes_by_definition_for_table('source_table_name_goes_here') + # drop_table('source_table_name_goes_here') + # rename_indexes_for_table('destination_table_name_goes_here', index_names) + def rename_indexes_for_table(table_name, new_index_names, schema_name: connection.current_schema) + current_index_names = indexes_by_definition_for_table(table_name, schema_name: schema_name) + rename_indexes(current_index_names, new_index_names, schema_name: schema_name) + end + private def find_indexes(table_name, schema_name: connection.current_schema) @@ -124,6 +162,18 @@ module Gitlab def generated_index_name(partition_name, index_name) object_name("#{partition_name}_#{index_name}", 'index') end + + def rename_indexes(from, to, schema_name: connection.current_schema) + indexes_to_rename = from.select { |index_id, _| to.has_key?(index_id) } + statements = indexes_to_rename.map do |index_id, index_name| + <<~SQL + ALTER INDEX #{connection.quote_table_name("#{schema_name}.#{connection.quote_column_name(index_name)}")} + RENAME TO #{connection.quote_column_name(to[index_id])} + SQL + end + + connection.execute(statements.join(';')) + end end end end diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb index 695a5d7ec77..f9790bf53b9 100644 --- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb +++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb @@ -275,7 +275,7 @@ module Gitlab ).revert_preparation_for_partitioning end - def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:) + def convert_table_to_first_list_partition(table_name:, partitioning_column:, parent_table_name:, initial_partitioning_value:, lock_tables: []) validate_not_in_transaction!(:convert_table_to_first_list_partition) Gitlab::Database::Partitioning::ConvertTableToFirstListPartition @@ -283,7 +283,8 @@ module Gitlab table_name: table_name, parent_table_name: parent_table_name, partitioning_column: partitioning_column, - zero_partition_value: initial_partitioning_value + zero_partition_value: initial_partitioning_value, + lock_tables: lock_tables ).partition end diff --git a/lib/gitlab/database/postgres_partition.rb b/lib/gitlab/database/postgres_partition.rb index eb080904f73..eda11fd8382 100644 --- a/lib/gitlab/database/postgres_partition.rb +++ b/lib/gitlab/database/postgres_partition.rb @@ -19,6 +19,20 @@ module Gitlab scope :for_parent_table, ->(name) { where("parent_identifier = concat(current_schema(), '.', ?)", name).order(:name) } + def self.partition_exists?(table_name) + where("identifier = concat(current_schema(), '.', ?)", table_name).exists? + end + + def self.legacy_partition_exists?(table_name) + result = connection.select_value(<<~SQL) + SELECT true FROM pg_class + WHERE relname = '#{table_name}' + AND relispartition = true; + SQL + + !!result + end + def to_s name end diff --git a/lib/gitlab/database/query_analyzer.rb b/lib/gitlab/database/query_analyzer.rb index 6f64d04270f..1280789b30c 100644 --- a/lib/gitlab/database/query_analyzer.rb +++ b/lib/gitlab/database/query_analyzer.rb @@ -86,7 +86,11 @@ module Gitlab analyzers.each do |analyzer| next if analyzer.suppressed? && !analyzer.requires_tracking?(parsed) - analyzer.analyze(parsed) + if analyzer.raw? + analyzer.analyze(sql) + else + analyzer.analyze(parsed) + end rescue StandardError, ::Gitlab::Database::QueryAnalyzers::Base::QueryAnalyzerError => e # We catch all standard errors to prevent validation errors to introduce fatal errors in production Gitlab::ErrorTracking.track_and_raise_for_dev_exception(e) diff --git a/lib/gitlab/database/query_analyzers/base.rb b/lib/gitlab/database/query_analyzers/base.rb index 9a52a4f6e23..9c2c228f869 100644 --- a/lib/gitlab/database/query_analyzers/base.rb +++ b/lib/gitlab/database/query_analyzers/base.rb @@ -53,6 +53,10 @@ module Gitlab Thread.current[self.context_key] end + def self.raw? + false + end + def self.enabled? raise NotImplementedError end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb new file mode 100644 index 00000000000..47277182d9a --- /dev/null +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_id_analyzer.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + module Ci + # The purpose of this analyzer is to detect queries missing partition_id clause + # when selecting, inserting, updating or deleting data. + class PartitioningIdAnalyzer < Database::QueryAnalyzers::Base + PartitionIdMissingError = Class.new(QueryAnalyzerError) + + ROUTING_TABLES = %w[p_ci_builds_metadata].freeze + + class << self + def enabled? + ::Feature::FlipperFeature.table_exists? && + ::Feature.enabled?(:ci_partitioning_analyze_queries_partition_id_check, type: :ops) + end + + def analyze(parsed) + analyze_partition_id_presence(parsed) + end + + private + + def analyze_partition_id_presence(parsed) + detected = ROUTING_TABLES & (parsed.pg.dml_tables + parsed.pg.select_tables) + return if detected.none? + + if insert_query?(parsed) + return if insert_include_partition_id?(parsed) + else + detected_with_selected_columns = parsed_detected_tables(parsed, detected) + return if partition_id_included?(detected_with_selected_columns) + end + + ::Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + PartitionIdMissingError.new( + "Detected query against a partitioned table without partition id: #{parsed.sql}" + ) + ) + end + + def parsed_detected_tables(parsed, routing_tables) + parsed.pg.filter_columns.each_with_object(Hash.new { |h, k| h[k] = [] }) do |item, hash| + table_name = item[0] || routing_tables[0] + column_name = item[1] + + hash[table_name] << column_name if routing_tables.include?(table_name) + end + end + + def partition_id_included?(result) + return false if result.empty? + + result.all? { |_routing_table, columns| columns.include?('partition_id') } + end + + def insert_query?(parsed) + parsed.sql.start_with?('INSERT') + end + + def insert_include_partition_id?(parsed) + filtered_columns_on_insert(parsed).include?('partition_id') + end + + def filtered_columns_on_insert(parsed) + result = parsed.pg.tree.to_h.dig(:stmts, 0, :stmt, :insert_stmt, :cols).map do |h| + h.dig(:res_target, :name) + end + + result || [] + end + end + end + end + end + end +end diff --git a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb index c2d5dfc1a15..eb55ebc7619 100644 --- a/lib/gitlab/database/query_analyzers/ci/partitioning_analyzer.rb +++ b/lib/gitlab/database/query_analyzers/ci/partitioning_routing_analyzer.rb @@ -5,12 +5,10 @@ module Gitlab module QueryAnalyzers module Ci # The purpose of this analyzer is to detect queries not going through a partitioning routing table - class PartitioningAnalyzer < Database::QueryAnalyzers::Base + class PartitioningRoutingAnalyzer < Database::QueryAnalyzers::Base RoutingTableNotUsedError = Class.new(QueryAnalyzerError) - ENABLED_TABLES = %w[ - ci_builds_metadata - ].freeze + ENABLED_TABLES = %w[ci_builds_metadata].freeze class << self def enabled? diff --git a/lib/gitlab/database/query_analyzers/query_recorder.rb b/lib/gitlab/database/query_analyzers/query_recorder.rb new file mode 100644 index 00000000000..88fe829c3d2 --- /dev/null +++ b/lib/gitlab/database/query_analyzers/query_recorder.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module QueryAnalyzers + class QueryRecorder < Base + LOG_FILE = 'rspec/query_recorder.ndjson' + + class << self + def raw? + true + end + + def enabled? + # Only enable QueryRecorder in CI + ENV['CI'].present? + end + + def analyze(sql) + payload = { + sql: sql + } + + log_query(payload) + end + + private + + def log_query(payload) + log_path = Rails.root.join(LOG_FILE) + log_dir = File.dirname(log_path) + + # Create log directory if it does not exist since it is only created + # ahead of time by certain CI jobs + FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir) + + log_line = "#{Gitlab::Json.dump(payload)}\n" + + File.write(log_path, log_line, mode: 'a') + end + end + end + end + end +end diff --git a/lib/gitlab/database/tables_truncate.rb b/lib/gitlab/database/tables_truncate.rb index 164520fbab3..8380bf23899 100644 --- a/lib/gitlab/database/tables_truncate.rb +++ b/lib/gitlab/database/tables_truncate.rb @@ -14,7 +14,7 @@ module Gitlab end def execute - raise "Cannot truncate legacy tables in single-db setup" unless Gitlab::Database.has_config?(:ci) + raise "Cannot truncate legacy tables in single-db setup" if single_database_setup? raise "database is not supported" unless %w[main ci].include?(database_name) logger&.info "DRY RUN:" if dry_run @@ -91,6 +91,13 @@ module Gitlab end end end + + def single_database_setup? + return true unless Gitlab::Database.has_config?(:ci) + + ci_base_model = Gitlab::Database.database_base_models[:ci] + !!Gitlab::Database.db_config_share_with(ci_base_model.connection_db_config) + end end end end diff --git a/lib/gitlab/database/type/symbolized_jsonb.rb b/lib/gitlab/database/type/symbolized_jsonb.rb new file mode 100644 index 00000000000..5bec738ec9c --- /dev/null +++ b/lib/gitlab/database/type/symbolized_jsonb.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +module Gitlab + module Database + module Type + # Extends Rails' Jsonb data type to deserialize it into symbolized Hash. + # + # Example: + # + # class SomeModel < ApplicationRecord + # # some_model.a_field is of type `jsonb` + # attribute :a_field, :sym_jsonb + # end + class SymbolizedJsonb < ::ActiveRecord::ConnectionAdapters::PostgreSQL::OID::Jsonb + def type + :sym_jsonb + end + + def deserialize(value) + data = super + return unless data + + ::Gitlab::Utils.deep_symbolized_access(data) + end + end + end + end +end diff --git a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb index 57d354eb907..be500171bef 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/create_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/create_service.rb @@ -98,7 +98,7 @@ module Gitlab if environment.save success(result) else - log_error("Could not create environment for the Self monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) + log_error("Could not create environment for the Self-monitoring project. Errors: %{errors}" % { errors: environment.errors.full_messages }) error(_('Could not create environment')) end end diff --git a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb index 998977b4000..d5bed94d735 100644 --- a/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb +++ b/lib/gitlab/database_importers/self_monitoring/project/delete_service.rb @@ -23,7 +23,7 @@ module Gitlab def validate_self_monitoring_project_exists(result) unless project_created? || self_monitoring_project_id.present? - return error(_('Self monitoring project does not exist')) + return error(_('Self-monitoring project does not exist')) end success(result) diff --git a/lib/gitlab/diff/file.rb b/lib/gitlab/diff/file.rb index 5583c896803..d5c0b187f92 100644 --- a/lib/gitlab/diff/file.rb +++ b/lib/gitlab/diff/file.rb @@ -44,10 +44,6 @@ module Gitlab add_blobs_to_batch_loader end - def use_semantic_ipynb_diff? - strong_memoize(:_use_semantic_ipynb_diff) { Feature.enabled?(:ipynb_semantic_diff, repository.project) } - end - def has_renderable? rendered&.has_renderable? end @@ -372,7 +368,7 @@ module Gitlab end def rendered - return unless use_semantic_ipynb_diff? && ipynb? && modified_file? && !collapsed? && !too_large? + return unless ipynb? && modified_file? && !collapsed? && !too_large? strong_memoize(:rendered) { Rendered::Notebook::DiffFile.new(self) } end diff --git a/lib/gitlab/diff/file_collection/base.rb b/lib/gitlab/diff/file_collection/base.rb index 924de132840..ae55dae1201 100644 --- a/lib/gitlab/diff/file_collection/base.rb +++ b/lib/gitlab/diff/file_collection/base.rb @@ -46,7 +46,9 @@ module Gitlab # This is either the new path, otherwise the old path for the diff_file def diff_file_paths - diff_files.map(&:file_path) + diffs.map do |diff| + diff.new_path.presence || diff.old_path + end end # This is both the new and old paths for the diff_file diff --git a/lib/gitlab/diff/highlight_cache.rb b/lib/gitlab/diff/highlight_cache.rb index d6f5e45c034..5128b09aef4 100644 --- a/lib/gitlab/diff/highlight_cache.rb +++ b/lib/gitlab/diff/highlight_cache.rb @@ -62,7 +62,7 @@ module Gitlab end def clear - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.del(key) end end @@ -124,7 +124,7 @@ module Gitlab # ...it will write/update a Gitlab::Redis hash (HSET) # def write_to_redis_hash(hash) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| hash.each do |diff_file_id, highlighted_diff_lines_hash| pipeline.hset( @@ -132,7 +132,7 @@ module Gitlab diff_file_id, gzip_compress(highlighted_diff_lines_hash.to_json) ) - rescue Encoding::UndefinedConversionError + rescue Encoding::UndefinedConversionError, EncodingError, JSON::GeneratorError nil end @@ -189,7 +189,7 @@ module Gitlab results = [] cache_key = key # Moving out redis calls for feature flags out of redis.pipelined - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| results = pipeline.hmget(cache_key, file_paths) pipeline.expire(key, EXPIRATION) @@ -223,6 +223,10 @@ module Gitlab ::Gitlab::Metrics::WebTransaction.current end + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end + def record_hit_ratio(results) current_transaction&.increment(:gitlab_redis_diff_caching_requests_total) current_transaction&.increment(:gitlab_redis_diff_caching_hits_total) if results.any?(&:present?) diff --git a/lib/gitlab/discussions_diff/highlight_cache.rb b/lib/gitlab/discussions_diff/highlight_cache.rb index 3337aeb9262..14cb773251b 100644 --- a/lib/gitlab/discussions_diff/highlight_cache.rb +++ b/lib/gitlab/discussions_diff/highlight_cache.rb @@ -14,12 +14,14 @@ module Gitlab # # mapping - Write multiple cache values at once def write_multiple(mapping) - Redis::Cache.with do |redis| - redis.multi do |multi| - mapping.each do |raw_key, value| - key = cache_key_for(raw_key) + with_redis do |redis| + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + mapping.each do |raw_key, value| + key = cache_key_for(raw_key) - multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) + multi.set(key, gzip_compress(value.to_json), ex: EXPIRATION) + end end end end @@ -37,7 +39,7 @@ module Gitlab keys = raw_keys.map { |id| cache_key_for(id) } content = - Redis::Cache.with do |redis| + with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.mget(keys) end @@ -62,7 +64,7 @@ module Gitlab keys = raw_keys.map { |id| cache_key_for(id) } - Redis::Cache.with do |redis| + with_redis do |redis| Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do redis.del(keys) end @@ -78,6 +80,10 @@ module Gitlab def cache_key_prefix "#{Redis::Cache::CACHE_NAMESPACE}:#{VERSION}:discussion-highlight" end + + def with_redis(&block) + Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb index f9e6d4076f3..7bb9ac2ffdb 100644 --- a/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb +++ b/lib/gitlab/doorkeeper_secret_storing/token/pbkdf2_sha512.rb @@ -12,8 +12,6 @@ module Gitlab SALT = '' def self.transform_secret(plain_secret) - return plain_secret unless Feature.enabled?(:hash_oauth_tokens) - Devise::Pbkdf2Encryptable::Encryptors::Pbkdf2Sha512.digest(plain_secret, STRETCHES, SALT) end diff --git a/lib/gitlab/email/common.rb b/lib/gitlab/email/common.rb new file mode 100644 index 00000000000..afee8d9cd3d --- /dev/null +++ b/lib/gitlab/email/common.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module Email + # Contains common methods which must be present in all email classes + module Common + UNSUBSCRIBE_SUFFIX = '-unsubscribe' + UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe' + WILDCARD_PLACEHOLDER = '%{key}' + + # This can be overridden for a custom config + def config + raise NotImplementedError + end + + def incoming_email_config + Gitlab.config.incoming_email + end + + def enabled? + !!config&.enabled && config.address.present? + end + + def supports_wildcard? + config_address = incoming_email_config.address + + config_address.present? && config_address.include?(WILDCARD_PLACEHOLDER) + end + + def supports_issue_creation? + enabled? && supports_wildcard? + end + + def reply_address(key) + incoming_email_config.address.sub(WILDCARD_PLACEHOLDER, key) + end + + # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com + def unsubscribe_address(key) + incoming_email_config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") + end + + def key_from_address(address, wildcard_address: nil) + raise NotImplementedError + end + + def key_from_fallback_message_id(mail_id) + message_id_regexp = /\Areply-(.+)@#{Gitlab.config.gitlab.host}\z/ + + mail_id[message_id_regexp, 1] + end + + def scan_fallback_references(references) + # It's looking for each <...> + references.scan(/(?!<)[^<>]+(?=>)/) + end + end + end +end diff --git a/lib/gitlab/email/handler/create_issue_handler.rb b/lib/gitlab/email/handler/create_issue_handler.rb index 434893eab82..e21a88c4e0d 100644 --- a/lib/gitlab/email/handler/create_issue_handler.rb +++ b/lib/gitlab/email/handler/create_issue_handler.rb @@ -73,7 +73,7 @@ module Gitlab end def can_handle_legacy_format? - project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY) + project_path && !incoming_email_token.include?('+') && !mail_key.include?(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX_LEGACY) end end end diff --git a/lib/gitlab/email/handler/unsubscribe_handler.rb b/lib/gitlab/email/handler/unsubscribe_handler.rb index 528857aff14..a4e526d9a24 100644 --- a/lib/gitlab/email/handler/unsubscribe_handler.rb +++ b/lib/gitlab/email/handler/unsubscribe_handler.rb @@ -12,8 +12,8 @@ module Gitlab delegate :project, to: :sent_notification, allow_nil: true HANDLER_REGEX_FOR = -> (suffix) { /\A(?<reply_token>\w+)#{Regexp.escape(suffix)}\z/ }.freeze - HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX).freeze - HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::IncomingEmail::UNSUBSCRIBE_SUFFIX_LEGACY).freeze + HANDLER_REGEX = HANDLER_REGEX_FOR.call(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX).freeze + HANDLER_REGEX_LEGACY = HANDLER_REGEX_FOR.call(Gitlab::Email::Common::UNSUBSCRIBE_SUFFIX_LEGACY).freeze def initialize(mail, mail_key) super(mail, mail_key) diff --git a/lib/gitlab/email/receiver.rb b/lib/gitlab/email/receiver.rb index ba84be6e8ca..1e03f5d17ee 100644 --- a/lib/gitlab/email/receiver.rb +++ b/lib/gitlab/email/receiver.rb @@ -10,6 +10,14 @@ module Gitlab RECEIVED_HEADER_REGEX = /for\s+\<(.+)\>/.freeze + # Errors that are purely from users and not anything we can control + USER_ERRORS = [ + Gitlab::Email::AutoGeneratedEmailError, Gitlab::Email::ProjectNotFound, Gitlab::Email::EmptyEmailError, + Gitlab::Email::UserNotFoundError, Gitlab::Email::UserBlockedError, Gitlab::Email::UserNotAuthorizedError, + Gitlab::Email::NoteableNotFoundError, Gitlab::Email::InvalidAttachment, Gitlab::Email::InvalidRecordError, + Gitlab::Email::EmailTooLarge + ].freeze + def initialize(raw) @raw = raw end @@ -24,6 +32,9 @@ module Gitlab handler.execute.tap do Gitlab::Metrics::BackgroundTransaction.current&.add_event(handler.metrics_event, handler.metrics_params) end + rescue *USER_ERRORS => e + # do not send a metric event since these are purely user errors that we can't control + raise e rescue StandardError => e Gitlab::Metrics::BackgroundTransaction.current&.add_event('email_receiver_error', error: e.class.name) raise e diff --git a/lib/gitlab/environment.rb b/lib/gitlab/environment.rb index 3c6ed696b9d..b1a9603d3a5 100644 --- a/lib/gitlab/environment.rb +++ b/lib/gitlab/environment.rb @@ -5,9 +5,5 @@ module Gitlab def self.hostname @hostname ||= ENV['HOSTNAME'] || Socket.gethostname end - - def self.qa_user_agent - ENV['GITLAB_QA_USER_AGENT'] - end end end diff --git a/lib/gitlab/error_tracking.rb b/lib/gitlab/error_tracking.rb index 83920182da4..582c3380869 100644 --- a/lib/gitlab/error_tracking.rb +++ b/lib/gitlab/error_tracking.rb @@ -131,6 +131,9 @@ module Gitlab end def before_send(event, hint) + # Don't report Sidekiq retry errors to Sentry + return if hint[:exception].is_a?(Gitlab::SidekiqMiddleware::RetryError) + inject_context_for_exception(event, hint[:exception]) custom_fingerprinting(event, hint[:exception]) diff --git a/lib/gitlab/etag_caching/store.rb b/lib/gitlab/etag_caching/store.rb index 437d577e70e..bc97c88ce85 100644 --- a/lib/gitlab/etag_caching/store.rb +++ b/lib/gitlab/etag_caching/store.rb @@ -15,10 +15,12 @@ module Gitlab def touch(*keys, only_if_missing: false) etags = keys.map { generate_etag } - Gitlab::Redis::SharedState.with do |redis| - redis.pipelined do |pipeline| - keys.each_with_index do |key, i| - pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + Gitlab::Redis::SharedState.with do |redis| + redis.pipelined do |pipeline| + keys.each_with_index do |key, i| + pipeline.set(redis_shared_state_key(key), etags[i], ex: EXPIRY_TIME, nx: only_if_missing) + end end end end diff --git a/lib/gitlab/experimentation/group_types.rb b/lib/gitlab/experimentation/group_types.rb deleted file mode 100644 index 8e8f7284b99..00000000000 --- a/lib/gitlab/experimentation/group_types.rb +++ /dev/null @@ -1,10 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module Experimentation - module GroupTypes - GROUP_CONTROL = :control - GROUP_EXPERIMENTAL = :experimental - end - end -end diff --git a/lib/gitlab/external_authorization/cache.rb b/lib/gitlab/external_authorization/cache.rb index c06711d16f8..2ba1a363421 100644 --- a/lib/gitlab/external_authorization/cache.rb +++ b/lib/gitlab/external_authorization/cache.rb @@ -11,7 +11,7 @@ module Gitlab end def load - @access, @reason, @refreshed_at = ::Gitlab::Redis::Cache.with do |redis| + @access, @reason, @refreshed_at = with_redis do |redis| redis.hmget(cache_key, :access, :reason, :refreshed_at) end @@ -19,7 +19,7 @@ module Gitlab end def store(new_access, new_reason, new_refreshed_at) - ::Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.pipelined do |pipeline| pipeline.mapped_hmset( cache_key, @@ -58,6 +58,10 @@ module Gitlab def cache_key "external_authorization:user-#{@user.id}:label-#{@label}" end + + def with_redis(&block) + ::Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/feature_categories.rb b/lib/gitlab/feature_categories.rb index d06f3b14fed..17586a94d7e 100644 --- a/lib/gitlab/feature_categories.rb +++ b/lib/gitlab/feature_categories.rb @@ -31,6 +31,14 @@ module Gitlab category end + def get!(feature_category) + return feature_category if valid?(feature_category) + + raise "Unknown feature category: #{feature_category}" if Gitlab.dev_or_test_env? + + FEATURE_CATEGORY_DEFAULT + end + def valid?(category) categories.include?(category.to_s) end diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index 9bbe17dcad1..b8f4ff0e9c4 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -45,7 +45,7 @@ module Gitlab # Relative path of repo attr_reader :relative_path - attr_reader :storage, :gl_repository, :gl_project_path + attr_reader :storage, :gl_repository, :gl_project_path, :container # This remote name has to be stable for all types of repositories that # can join an object pool. If it's structure ever changes, a migration @@ -56,11 +56,12 @@ module Gitlab # This initializer method is only used on the client side (gitlab-ce). # Gitaly-ruby uses a different initializer. - def initialize(storage, relative_path, gl_repository, gl_project_path) + def initialize(storage, relative_path, gl_repository, gl_project_path, container: nil) @storage = storage @relative_path = relative_path @gl_repository = gl_repository @gl_project_path = gl_project_path + @container = container @name = @relative_path.split("/").last end @@ -69,6 +70,11 @@ module Gitlab "<#{self.class.name}: #{self.gl_project_path}>" end + # Support Feature Flag Repository actor + def flipper_id + "Repository:#{@relative_path}" + end + def ==(other) other.is_a?(self.class) && [storage, relative_path] == [other.storage, other.relative_path] end @@ -534,9 +540,9 @@ module Gitlab # Returns matching refs for OID # # Limit of 0 means there is no limit. - def refs_by_oid(oid:, limit: 0) + def refs_by_oid(oid:, limit: 0, ref_patterns: nil) wrapped_gitaly_errors do - gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit) + gitaly_ref_client.find_refs_by_oid(oid: oid, limit: limit, ref_patterns: ref_patterns) end rescue CommandError, TypeError, NoRepository nil @@ -1054,19 +1060,19 @@ module Gitlab end end - def search_files_by_name(query, ref) + def search_files_by_name(query, ref, limit: 0, offset: 0) safe_query = query.sub(%r{^/*}, "") ref ||= root_ref return [] if empty? || safe_query.blank? - gitaly_repository_client.search_files_by_name(ref, safe_query).map do |file| + gitaly_repository_client.search_files_by_name(ref, safe_query, limit: limit, offset: offset).map do |file| Gitlab::EncodingHelper.encode_utf8(file) end end - def search_files_by_regexp(filter, ref = 'HEAD') - gitaly_repository_client.search_files_by_regexp(ref, filter).map do |file| + def search_files_by_regexp(filter, ref = 'HEAD', limit: 0, offset: 0) + gitaly_repository_client.search_files_by_regexp(ref, filter, limit: limit, offset: offset).map do |file| Gitlab::EncodingHelper.encode_utf8(file) end end diff --git a/lib/gitlab/git_ref_validator.rb b/lib/gitlab/git_ref_validator.rb index 1330b06bf9c..f4d4cebc096 100644 --- a/lib/gitlab/git_ref_validator.rb +++ b/lib/gitlab/git_ref_validator.rb @@ -13,6 +13,7 @@ module Gitlab # # Returns true for a valid reference name, false otherwise def validate(ref_name) + return false if ref_name.to_s.empty? # #blank? raises an ArgumentError for invalid encodings return false if ref_name.start_with?(*(EXPANDED_PREFIXES + DISALLOWED_PREFIXES)) return false if ref_name == 'HEAD' @@ -24,6 +25,7 @@ module Gitlab end def validate_merge_request_branch(ref_name) + return false if ref_name.to_s.empty? return false if ref_name.start_with?(*DISALLOWED_PREFIXES) expanded_name = if ref_name.start_with?(*EXPANDED_PREFIXES) diff --git a/lib/gitlab/gitaly_client.rb b/lib/gitlab/gitaly_client.rb index 996534f4194..735c7fcf80c 100644 --- a/lib/gitlab/gitaly_client.rb +++ b/lib/gitlab/gitaly_client.rb @@ -204,8 +204,9 @@ module Gitlab metadata['x-gitlab-correlation-id'] = Labkit::Correlation::CorrelationId.current_id if Labkit::Correlation::CorrelationId.current_id metadata['gitaly-session-id'] = session_id metadata['username'] = context_data['meta.user'] if context_data&.fetch('meta.user', nil) + metadata['user_id'] = context_data['meta.user_id'].to_s if context_data&.fetch('meta.user_id', nil) metadata['remote_ip'] = context_data['meta.remote_ip'] if context_data&.fetch('meta.remote_ip', nil) - metadata.merge!(Feature::Gitaly.server_feature_flags) + metadata.merge!(Feature::Gitaly.server_feature_flags(**feature_flag_actors)) metadata.merge!(route_to_primary) deadline_info = request_deadline(timeout) @@ -293,7 +294,7 @@ module Gitlab # check if the limit is being exceeded while testing in those environments # In that case we can use a feature flag to indicate that we do want to # enforce request limits. - return true if Feature::Gitaly.enabled?('enforce_requests_limits') + return true if Feature::Gitaly.enabled_for_any?(:gitaly_enforce_requests_limits) !Rails.env.production? end @@ -502,5 +503,24 @@ module Gitlab end private_class_method :max_stacks + + def self.with_feature_flag_actors(repository: nil, user: nil, project: nil, group: nil, &block) + feature_flag_actors[:repository] = repository + feature_flag_actors[:user] = user + feature_flag_actors[:project] = project + feature_flag_actors[:group] = group + + yield + ensure + feature_flag_actors.clear + end + + def self.feature_flag_actors + if Gitlab::SafeRequestStore.active? + Gitlab::SafeRequestStore[:gitaly_feature_flag_actors] ||= {} + else + Thread.current[:gitaly_feature_flag_actors] ||= {} + end + end end end diff --git a/lib/gitlab/gitaly_client/blob_service.rb b/lib/gitlab/gitaly_client/blob_service.rb index 3b08a833aeb..6d87c3329d7 100644 --- a/lib/gitlab/gitaly_client/blob_service.rb +++ b/lib/gitlab/gitaly_client/blob_service.rb @@ -4,9 +4,12 @@ module Gitlab module GitalyClient class BlobService include Gitlab::EncodingHelper + include WithFeatureFlagActors def initialize(repository) @gitaly_repo = repository.gitaly_repository + + self.repository_actor = repository end def get_blob(oid:, limit:) @@ -15,7 +18,7 @@ module Gitlab oid: oid, limit: limit ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :get_blob, request, timeout: GitalyClient.fast_timeout) consume_blob_response(response) end @@ -35,7 +38,7 @@ module Gitlab GitalyClient.medium_timeout end - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :list_blobs, request, timeout: timeout) GitalyClient::BlobsStitcher.new(GitalyClient::ListBlobsAdapter.new(response)) end @@ -47,7 +50,7 @@ module Gitlab blob_ids: blob_ids ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :get_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end @@ -64,7 +67,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, :get_blobs, @@ -87,7 +90,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, :get_blobs, @@ -107,7 +110,7 @@ module Gitlab GitalyClient.medium_timeout end - response = GitalyClient.call( + response = gitaly_client_call( @gitaly_repo.storage_name, :blob_service, rpc, @@ -123,7 +126,7 @@ module Gitlab revisions: [encode_binary("--all")] ) - response = GitalyClient.call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@gitaly_repo.storage_name, :blob_service, :list_lfs_pointers, request, timeout: GitalyClient.medium_timeout) map_lfs_pointers(response) end diff --git a/lib/gitlab/gitaly_client/cleanup_service.rb b/lib/gitlab/gitaly_client/cleanup_service.rb index 649aaa46362..3c2c41a244e 100644 --- a/lib/gitlab/gitaly_client/cleanup_service.rb +++ b/lib/gitlab/gitaly_client/cleanup_service.rb @@ -3,6 +3,8 @@ module Gitlab module GitalyClient class CleanupService + include WithFeatureFlagActors + attr_reader :repository, :gitaly_repo, :storage # 'repository' is a Gitlab::Git::Repository @@ -10,10 +12,12 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def apply_bfg_object_map_stream(io, &blk) - response = GitalyClient.call( + response = gitaly_client_call( storage, :cleanup_service, :apply_bfg_object_map_stream, diff --git a/lib/gitlab/gitaly_client/commit_service.rb b/lib/gitlab/gitaly_client/commit_service.rb index 312d1dddff1..6bcf4802fbe 100644 --- a/lib/gitlab/gitaly_client/commit_service.rb +++ b/lib/gitlab/gitaly_client/commit_service.rb @@ -4,12 +4,15 @@ module Gitlab module GitalyClient class CommitService include Gitlab::EncodingHelper + include WithFeatureFlagActors TREE_ENTRIES_DEFAULT_LIMIT = 100_000 def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository + + self.repository_actor = repository end def ls_files(revision) @@ -18,7 +21,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_files, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map { |d| EncodingHelper.encode!(d.dup) } end @@ -31,7 +34,7 @@ module Gitlab child_id: child_id ) - GitalyClient.call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value + gitaly_client_call(@repository.storage, :commit_service, :commit_is_ancestor, request, timeout: GitalyClient.fast_timeout).value end def diff(from, to, options = {}) @@ -74,7 +77,7 @@ module Gitlab def commit_deltas(commit) request = Gitaly::CommitDeltaRequest.new(diff_from_parent_request_params(commit)) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_delta, request, timeout: GitalyClient.fast_timeout) response.flat_map { |msg| msg.deltas } end @@ -93,7 +96,7 @@ module Gitlab limit: limit.to_i ) - response = GitalyClient.call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :tree_entry, request, timeout: GitalyClient.medium_timeout) entry = nil data = [] @@ -127,7 +130,7 @@ module Gitlab ) request.sort = Gitaly::GetTreeEntriesRequest::SortBy::TREES_FIRST if pagination_params - response = GitalyClient.call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_tree_entries, request, timeout: GitalyClient.medium_timeout) cursor = nil @@ -163,7 +166,7 @@ module Gitlab request.path = encode_binary(options[:path]) if options[:path].present? request.max_count = options[:max_count] if options[:max_count].present? - GitalyClient.call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count + gitaly_client_call(@repository.storage, :commit_service, :count_commits, request, timeout: GitalyClient.medium_timeout).count end def diverging_commit_count(from, to, max_count:) @@ -173,7 +176,7 @@ module Gitlab to: encode_binary(to), max_count: max_count ) - response = GitalyClient.call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :count_diverging_commits, request, timeout: GitalyClient.medium_timeout) [response.left_count, response.right_count] end @@ -187,7 +190,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_last_commits_for_tree, request, timeout: GitalyClient.medium_timeout) response.each_with_object({}) do |gitaly_response, hsh| gitaly_response.commits.each do |commit_for_tree| @@ -204,7 +207,7 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - gitaly_commit = GitalyClient.call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit + gitaly_commit = gitaly_client_call(@repository.storage, :commit_service, :last_commit_for_path, request, timeout: GitalyClient.fast_timeout).commit return unless gitaly_commit Gitlab::Git::Commit.new(@repository, gitaly_commit) @@ -217,7 +220,7 @@ module Gitlab right_commit_id: right_commit_sha ) - response = GitalyClient.call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :diff_stats, request, timeout: GitalyClient.medium_timeout) response.flat_map { |rsp| rsp.stats.to_a } end @@ -227,7 +230,7 @@ module Gitlab commits: commits ) - response = GitalyClient.call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :find_changed_paths, request, timeout: GitalyClient.medium_timeout) response.flat_map do |msg| msg.paths.map do |path| Gitlab::Git::ChangedPath.new( @@ -247,7 +250,7 @@ module Gitlab ) request.order = opts[:order].upcase if opts[:order].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_all_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -268,7 +271,7 @@ module Gitlab request.before = GitalyClient.timestamp(params[:before]) if params[:before] request.after = GitalyClient.timestamp(params[:after]) if params[:after] - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -290,7 +293,7 @@ module Gitlab repository: quarantined_repo ) - response = GitalyClient.call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_all_commits, request, timeout: GitalyClient.medium_timeout) quarantined_commits = consume_commits_response(response) quarantined_commit_ids = quarantined_commits.map(&:id) @@ -328,7 +331,7 @@ module Gitlab request = Gitaly::ListCommitsByOidRequest.new(repository: @gitaly_repo, oid: oids) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_oid, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) rescue GRPC::NotFound # If no repository is found, happens mainly during testing [] @@ -345,13 +348,13 @@ module Gitlab global_options: parse_global_options!(literal_pathspec: literal_pathspec) ) - response = GitalyClient.call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commits_by_message, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end def languages(ref = nil) request = Gitaly::CommitLanguagesRequest.new(repository: @gitaly_repo, revision: ref || '') - response = GitalyClient.call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :commit_languages, request, timeout: GitalyClient.long_timeout) response.languages.map { |l| { value: l.share.round(2), label: l.name, color: l.color, highlight: l.color } } end @@ -364,7 +367,7 @@ module Gitlab range: (encode_binary(range) if range) ) - response = GitalyClient.call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :raw_blame, request, timeout: GitalyClient.medium_timeout) response.reduce([]) { |memo, msg| memo << msg.data }.join end @@ -400,7 +403,7 @@ module Gitlab repository: @gitaly_repo, revision: encode_binary(revision) ) - GitalyClient.call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) + gitaly_client_call(@repository.storage, :commit_service, :commit_stats, request, timeout: GitalyClient.medium_timeout) end def find_commits(options) @@ -424,7 +427,7 @@ module Gitlab request.paths = encode_repeated(Array(options[:path])) if options[:path].present? - response = GitalyClient.call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commits, request, timeout: GitalyClient.medium_timeout) consume_commits_response(response) end @@ -443,7 +446,7 @@ module Gitlab end end - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :commit_service, :check_objects_exist, enum, timeout: GitalyClient.medium_timeout ) @@ -470,7 +473,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :filter_shas_with_signatures, enum, timeout: GitalyClient.fast_timeout) response.flat_map do |msg| msg.shas.map { |sha| EncodingHelper.encode!(sha) } end @@ -478,7 +481,7 @@ module Gitlab def get_commit_signatures(commit_ids) request = Gitaly::GetCommitSignaturesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_commit_id = nil @@ -497,7 +500,7 @@ module Gitlab def get_commit_messages(commit_ids) request = Gitaly::GetCommitMessagesRequest.new(repository: @gitaly_repo, commit_ids: commit_ids) - response = GitalyClient.call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :get_commit_messages, request, timeout: GitalyClient.fast_timeout) messages = Hash.new { |h, k| h[k] = +''.b } current_commit_id = nil @@ -515,7 +518,7 @@ module Gitlab request = Gitaly::ListCommitsByRefNameRequest .new(repository: @gitaly_repo, ref_names: refs.map { |ref| encode_binary(ref) }) - response = GitalyClient.call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :list_commits_by_ref_name, request, timeout: GitalyClient.medium_timeout) commit_refs = response.flat_map do |message| message.commit_refs.map do |commit_ref| @@ -540,7 +543,7 @@ module Gitlab request_params.merge!(Gitlab::Git::DiffCollection.limits(options)) request = Gitaly::CommitDiffRequest.new(request_params) - response = GitalyClient.call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :diff_service, :commit_diff, request, timeout: GitalyClient.medium_timeout) GitalyClient::DiffStitcher.new(response) end @@ -577,7 +580,7 @@ module Gitlab revision: encode_binary(revision) ) - response = GitalyClient.call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :commit_service, :find_commit, request, timeout: GitalyClient.medium_timeout) response.commit end diff --git a/lib/gitlab/gitaly_client/conflicts_service.rb b/lib/gitlab/gitaly_client/conflicts_service.rb index 982454b117e..38f648ccc31 100644 --- a/lib/gitlab/gitaly_client/conflicts_service.rb +++ b/lib/gitlab/gitaly_client/conflicts_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class ConflictsService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze @@ -12,6 +13,8 @@ module Gitlab @repository = repository @our_commit_oid = our_commit_oid @their_commit_oid = their_commit_oid + + self.repository_actor = repository end def list_conflict_files(allow_tree_conflicts: false) @@ -21,7 +24,7 @@ module Gitlab their_commit_oid: @their_commit_oid, allow_tree_conflicts: allow_tree_conflicts ) - response = GitalyClient.call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :conflicts_service, :list_conflict_files, request, timeout: GitalyClient.long_timeout) GitalyClient::ConflictFilesStitcher.new(response, @gitaly_repo) end @@ -50,7 +53,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :conflicts_service, :resolve_conflicts, req_enum, remote_storage: target_repository.storage, timeout: GitalyClient.long_timeout) if response.resolution_error.present? raise Gitlab::Git::Conflict::Resolver::ResolutionError, response.resolution_error diff --git a/lib/gitlab/gitaly_client/object_pool_service.rb b/lib/gitlab/gitaly_client/object_pool_service.rb index 786ef0ebebe..e07bf3fbccc 100644 --- a/lib/gitlab/gitaly_client/object_pool_service.rb +++ b/lib/gitlab/gitaly_client/object_pool_service.rb @@ -3,6 +3,8 @@ module Gitlab module GitalyClient class ObjectPoolService + include WithFeatureFlagActors + attr_reader :object_pool, :storage def initialize(object_pool) @@ -15,8 +17,10 @@ module Gitlab object_pool: object_pool, origin: repository.gitaly_repository) - GitalyClient.call(storage, :object_pool_service, :create_object_pool, - request, timeout: GitalyClient.medium_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :create_object_pool, + request, timeout: GitalyClient.medium_timeout) + end end def delete @@ -32,8 +36,10 @@ module Gitlab repository: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, - request, timeout: GitalyClient.fast_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :link_repository_to_object_pool, + request, timeout: GitalyClient.fast_timeout) + end end def fetch(repository) @@ -42,8 +48,10 @@ module Gitlab origin: repository.gitaly_repository ) - GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, - request, timeout: GitalyClient.long_timeout) + GitalyClient.with_feature_flag_actors(**gitaly_feature_flag_actors(repository)) do + GitalyClient.call(storage, :object_pool_service, :fetch_into_object_pool, + request, timeout: GitalyClient.long_timeout) + end end end end diff --git a/lib/gitlab/gitaly_client/operation_service.rb b/lib/gitlab/gitaly_client/operation_service.rb index 7835fb32f59..2312def5efc 100644 --- a/lib/gitlab/gitaly_client/operation_service.rb +++ b/lib/gitlab/gitaly_client/operation_service.rb @@ -4,12 +4,15 @@ module Gitlab module GitalyClient class OperationService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze def initialize(repository) @gitaly_repo = repository.gitaly_repository @repository = repository + + self.repository_actor = repository end def rm_tag(tag_name, user) @@ -19,7 +22,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :operation_service, :user_delete_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error @@ -36,7 +39,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@repository.storage, :operation_service, :user_create_tag, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence raise Gitlab::Git::PreReceiveError, pre_receive_error elsif response.exists @@ -73,7 +76,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly, start_point: encode_binary(start_point) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_create_branch, request, timeout: GitalyClient.long_timeout) if response.pre_receive_error.present? @@ -110,7 +113,7 @@ module Gitlab oldrev: encode_binary(oldrev) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_update_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence @@ -125,7 +128,7 @@ module Gitlab user: Gitlab::Git::User.from_gitlab(user).to_gitaly ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_delete_branch, request, timeout: GitalyClient.long_timeout) if pre_receive_error = response.pre_receive_error.presence @@ -156,7 +159,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_merge_to_ref, request, timeout: GitalyClient.long_timeout) response.commit_id @@ -164,7 +167,7 @@ module Gitlab def user_merge_branch(user, source_sha, target_branch, message) request_enum = QueueEnumerator.new - response_enum = GitalyClient.call( + response_enum = gitaly_client_call( @repository.storage, :operation_service, :user_merge_branch, @@ -225,7 +228,7 @@ module Gitlab branch: encode_binary(target_branch) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_ff_branch, @@ -268,7 +271,7 @@ module Gitlab request_enum = QueueEnumerator.new rebase_sha = nil - response_enum = GitalyClient.call( + response_enum = gitaly_client_call( @repository.storage, :operation_service, :user_rebase_confirmable, @@ -334,7 +337,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: time.to_i) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_squash, @@ -376,7 +379,7 @@ module Gitlab timestamp: Google::Protobuf::Timestamp.new(seconds: Time.now.utc.to_i) ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_update_submodule, @@ -422,7 +425,7 @@ module Gitlab end end - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :user_commit_files, req_enum, timeout: GitalyClient.long_timeout, remote_storage: start_repository&.storage) @@ -435,9 +438,25 @@ module Gitlab end Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) + rescue GRPC::BadStatus => e + detailed_error = GitalyClient.decode_detailed_error(e) + + case detailed_error&.error + when :access_check + access_check_error = detailed_error.access_check + # These messages were returned from internal/allowed API calls + raise Gitlab::Git::PreReceiveError.new(fallback_message: access_check_error.error_message) + when :custom_hook + raise Gitlab::Git::PreReceiveError.new(custom_hook_error_message(detailed_error.custom_hook), + fallback_message: e.details) + when :index_update + raise Gitlab::Git::Index::IndexError, index_error_message(detailed_error.index_update) + else + raise e + end end - # rubocop:enable Metrics/ParameterLists + # rubocop:enable Metrics/ParameterLists def user_commit_patches(user, branch_name, patches) header = Gitaly::UserApplyPatchRequest::Header.new( repository: @gitaly_repo, @@ -457,7 +476,7 @@ module Gitlab end end - response = GitalyClient.call(@repository.storage, :operation_service, + response = gitaly_client_call(@repository.storage, :operation_service, :user_apply_patch, chunks, timeout: GitalyClient.long_timeout) Gitlab::Git::OperationService::BranchUpdate.from_gitaly(response.branch_update) @@ -493,7 +512,7 @@ module Gitlab dry_run: dry_run ) - response = GitalyClient.call( + response = gitaly_client_call( @repository.storage, :operation_service, :"user_#{rpc}", @@ -575,6 +594,27 @@ module Gitlab custom_hook_output = custom_hook_error.stderr.presence || custom_hook_error.stdout EncodingHelper.encode!(custom_hook_output) end + + def index_error_message(index_error) + encoded_path = EncodingHelper.encode!(index_error.path) + + case index_error.error_type + when :ERROR_TYPE_EMPTY_PATH + "Received empty path" + when :ERROR_TYPE_INVALID_PATH + "Invalid path: #{encoded_path}" + when :ERROR_TYPE_DIRECTORY_EXISTS + "Directory already exists: #{encoded_path}" + when :ERROR_TYPE_DIRECTORY_TRAVERSAL + "Directory traversal in path escapes repository: #{encoded_path}" + when :ERROR_TYPE_FILE_EXISTS + "File already exists: #{encoded_path}" + when :ERROR_TYPE_FILE_NOT_FOUND + "File not found: #{encoded_path}" + else + "Unknown error performing git operation" + end + end end end end diff --git a/lib/gitlab/gitaly_client/praefect_info_service.rb b/lib/gitlab/gitaly_client/praefect_info_service.rb index 127f8cfbdf6..b565898acf8 100644 --- a/lib/gitlab/gitaly_client/praefect_info_service.rb +++ b/lib/gitlab/gitaly_client/praefect_info_service.rb @@ -3,16 +3,20 @@ module Gitlab module GitalyClient class PraefectInfoService + include WithFeatureFlagActors + def initialize(repository) @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def replicas request = Gitaly::RepositoryReplicasRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :praefect_info_service, :repository_replicas, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :praefect_info_service, :repository_replicas, request, timeout: GitalyClient.fast_timeout) end end end diff --git a/lib/gitlab/gitaly_client/ref_service.rb b/lib/gitlab/gitaly_client/ref_service.rb index d2b702f3a6d..de76ade76cb 100644 --- a/lib/gitlab/gitaly_client/ref_service.rb +++ b/lib/gitlab/gitaly_client/ref_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RefService include Gitlab::EncodingHelper + include WithFeatureFlagActors TAGS_SORT_KEY = { 'name' => Gitaly::FindAllTagsRequest::SortBy::Key::REFNAME, @@ -21,17 +22,19 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def branches request = Gitaly::FindAllBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def remote_branches(remote_name) request = Gitaly::FindAllRemoteBranchesRequest.new(repository: @gitaly_repo, remote_name: remote_name) - response = GitalyClient.call(@storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_remote_branches, request, timeout: GitalyClient.medium_timeout) consume_find_all_remote_branches_response(remote_name, response) end @@ -41,25 +44,25 @@ module Gitlab merged_only: true, merged_branches: branch_names.map { |s| encode_binary(s) } ) - response = GitalyClient.call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branches, request, timeout: GitalyClient.fast_timeout) consume_find_all_branches_response(response) end def default_branch_name request = Gitaly::FindDefaultBranchNameRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_default_branch_name, request, timeout: GitalyClient.fast_timeout) Gitlab::Git.branch_name(response.name) end def branch_names request = Gitaly::FindAllBranchNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_branch_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.branch_name(name) } end def tag_names request = Gitaly::FindAllTagNamesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_tag_names, request, timeout: GitalyClient.fast_timeout) consume_refs_response(response) { |name| Gitlab::Git.tag_name(name) } end @@ -74,7 +77,7 @@ module Gitlab def local_branches(sort_by: nil, pagination_params: nil) request = Gitaly::FindLocalBranchesRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_local_branches_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_local_branches, request, timeout: GitalyClient.fast_timeout) consume_find_local_branches_response(response) end @@ -82,13 +85,13 @@ module Gitlab request = Gitaly::FindAllTagsRequest.new(repository: @gitaly_repo, pagination_params: pagination_params) request.sort_by = sort_tags_by_param(sort_by) if sort_by - response = GitalyClient.call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_all_tags, request, timeout: GitalyClient.medium_timeout) consume_tags_response(response) end def ref_exists?(ref_name) request = Gitaly::RefExistsRequest.new(repository: @gitaly_repo, ref: encode_binary(ref_name)) - response = GitalyClient.call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :ref_exists, request, timeout: GitalyClient.fast_timeout) response.value rescue GRPC::InvalidArgument => e raise ArgumentError, e.message @@ -100,7 +103,7 @@ module Gitlab name: encode_binary(branch_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :find_branch, request, timeout: GitalyClient.medium_timeout) branch = response.branch return unless branch @@ -116,7 +119,7 @@ module Gitlab tag_name: encode_binary(tag_name) ) - response = GitalyClient.call(@repository.storage, :ref_service, :find_tag, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :find_tag, request, timeout: GitalyClient.medium_timeout) tag = response.tag return unless tag @@ -140,7 +143,7 @@ module Gitlab except_with_prefix: except_with_prefixes.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :delete_refs, request, timeout: GitalyClient.medium_timeout) raise Gitlab::Git::Repository::GitError, response.git_error if response.git_error.present? rescue GRPC::BadStatus => e @@ -164,7 +167,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call(@storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_tag_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(response, :tag_names) end @@ -176,7 +179,7 @@ module Gitlab limit: limit ) - response = GitalyClient.call(@storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_branch_names_containing_commit, request, timeout: GitalyClient.medium_timeout) consume_ref_contains_sha_response(response, :branch_names) end @@ -185,7 +188,7 @@ module Gitlab messages = Hash.new { |h, k| h[k] = +''.b } current_tag_id = nil - response = GitalyClient.call(@storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :get_tag_messages, request, timeout: GitalyClient.fast_timeout) response.each do |rpc_message| current_tag_id = rpc_message.tag_id if rpc_message.tag_id.present? @@ -197,7 +200,7 @@ module Gitlab def get_tag_signatures(tag_ids) request = Gitaly::GetTagSignaturesRequest.new(repository: @gitaly_repo, tag_revisions: tag_ids) - response = GitalyClient.call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@repository.storage, :ref_service, :get_tag_signatures, request, timeout: GitalyClient.fast_timeout) signatures = Hash.new { |h, k| h[k] = [+''.b, +''.b] } current_tag_id = nil @@ -222,20 +225,20 @@ module Gitlab patterns: patterns ) - response = GitalyClient.call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :ref_service, :list_refs, request, timeout: GitalyClient.fast_timeout) consume_list_refs_response(response) end def pack_refs request = Gitaly::PackRefsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :ref_service, :pack_refs, request, timeout: GitalyClient.long_timeout) end - def find_refs_by_oid(oid:, limit:) - request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit) + def find_refs_by_oid(oid:, limit:, ref_patterns: nil) + request = Gitaly::FindRefsByOIDRequest.new(repository: @gitaly_repo, sort_field: :refname, oid: oid, limit: limit, ref_patterns: ref_patterns) - response = GitalyClient.call(@storage, :ref_service, :find_refs_by_oid, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :ref_service, :find_refs_by_oid, request, timeout: GitalyClient.medium_timeout) response&.refs&.to_a end diff --git a/lib/gitlab/gitaly_client/remote_service.rb b/lib/gitlab/gitaly_client/remote_service.rb index 535b987f91c..9647cfad76e 100644 --- a/lib/gitlab/gitaly_client/remote_service.rb +++ b/lib/gitlab/gitaly_client/remote_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RemoteService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes.freeze @@ -24,6 +25,8 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def find_remote_root_ref(remote_url, authorization) @@ -31,7 +34,7 @@ module Gitlab remote_url: remote_url, http_authorization_header: authorization) - response = GitalyClient.call(@storage, :remote_service, + response = gitaly_client_call(@storage, :remote_service, :find_remote_root_ref, request, timeout: GitalyClient.medium_timeout) encode_utf8(response.ref) diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index f11437552e1..e6565bd33c2 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -4,6 +4,7 @@ module Gitlab module GitalyClient class RepositoryService include Gitlab::EncodingHelper + include WithFeatureFlagActors MAX_MSG_SIZE = 128.kilobytes @@ -11,57 +12,59 @@ module Gitlab @repository = repository @gitaly_repo = repository.gitaly_repository @storage = repository.storage + + self.repository_actor = repository end def exists? request = Gitaly::RepositoryExistsRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :repository_exists, request, timeout: GitalyClient.fast_timeout) response.exists end def optimize_repository request = Gitaly::OptimizeRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :optimize_repository, request, timeout: GitalyClient.long_timeout) end def prune_unreachable_objects request = Gitaly::PruneUnreachableObjectsRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :prune_unreachable_objects, request, timeout: GitalyClient.long_timeout) end def garbage_collect(create_bitmap, prune:) request = Gitaly::GarbageCollectRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap, prune: prune) - GitalyClient.call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :garbage_collect, request, timeout: GitalyClient.long_timeout) end def repack_full(create_bitmap) request = Gitaly::RepackFullRequest.new(repository: @gitaly_repo, create_bitmap: create_bitmap) - GitalyClient.call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :repack_full, request, timeout: GitalyClient.long_timeout) end def repack_incremental request = Gitaly::RepackIncrementalRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :repack_incremental, request, timeout: GitalyClient.long_timeout) end def repository_size request = Gitaly::RepositorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@storage, :repository_service, :repository_size, request, timeout: GitalyClient.long_timeout) response.size end def get_object_directory_size request = Gitaly::GetObjectDirectorySizeRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) + response = gitaly_client_call(@storage, :repository_service, :get_object_directory_size, request, timeout: GitalyClient.medium_timeout) response.size end def apply_gitattributes(revision) request = Gitaly::ApplyGitattributesRequest.new(repository: @gitaly_repo, revision: encode_binary(revision)) - GitalyClient.call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :apply_gitattributes, request, timeout: GitalyClient.fast_timeout) rescue GRPC::InvalidArgument => ex raise Gitlab::Git::Repository::InvalidRef, ex end @@ -69,7 +72,7 @@ module Gitlab def info_attributes request = Gitaly::GetInfoAttributesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :get_info_attributes, request, timeout: GitalyClient.fast_timeout) response.each_with_object([]) do |message, attributes| attributes << message.attributes end.join @@ -103,18 +106,18 @@ module Gitlab end end - GitalyClient.call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :fetch_remote, request, timeout: GitalyClient.long_timeout) end # rubocop: enable Metrics/ParameterLists def create_repository(default_branch = nil) request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: default_branch) - GitalyClient.call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end def has_local_branches? request = Gitaly::HasLocalBranchesRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :has_local_branches, request, timeout: GitalyClient.fast_timeout) response.value end @@ -125,7 +128,7 @@ module Gitlab revisions: revisions.map { |r| encode_binary(r) } ) - response = GitalyClient.call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :find_merge_base, request, timeout: GitalyClient.fast_timeout) response.base.presence end @@ -135,7 +138,7 @@ module Gitlab source_repository: source_repository.gitaly_repository ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_fork, @@ -153,7 +156,7 @@ module Gitlab mirror: mirror ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_repository_from_url, @@ -170,7 +173,7 @@ module Gitlab target_ref: local_ref.b ) - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, :fetch_source_branch, @@ -184,7 +187,7 @@ module Gitlab def fsck request = Gitaly::FsckRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) + response = gitaly_client_call(@storage, :repository_service, :fsck, request, timeout: GitalyClient.long_timeout) if response.error.empty? ["", 0] @@ -236,7 +239,7 @@ module Gitlab http_auth: http_auth ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :create_repository_from_snapshot, @@ -253,11 +256,11 @@ module Gitlab ) request.old_revision = old_ref.b unless old_ref.nil? - GitalyClient.call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :write_ref, request, timeout: GitalyClient.fast_timeout) end def set_full_path(path) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :set_full_path, @@ -272,7 +275,7 @@ module Gitlab end def full_path - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, :full_path, @@ -286,12 +289,12 @@ module Gitlab def find_license request = Gitaly::FindLicenseRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) + gitaly_client_call(@storage, :repository_service, :find_license, request, timeout: GitalyClient.medium_timeout) end def calculate_checksum request = Gitaly::CalculateChecksumRequest.new(repository: @gitaly_repo) - response = GitalyClient.call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) + response = gitaly_client_call(@storage, :repository_service, :calculate_checksum, request, timeout: GitalyClient.fast_timeout) response.checksum.presence rescue GRPC::DataLoss => e raise Gitlab::Git::Repository::InvalidRepository, e @@ -300,23 +303,23 @@ module Gitlab def raw_changes_between(from, to) request = Gitaly::GetRawChangesRequest.new(repository: @gitaly_repo, from_revision: from, to_revision: to) - GitalyClient.call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :get_raw_changes, request, timeout: GitalyClient.fast_timeout) end - def search_files_by_name(ref, query) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + def search_files_by_name(ref, query, limit: 0, offset: 0) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: query, limit: limit, offset: offset) + gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def search_files_by_content(ref, query, options = {}) request = Gitaly::SearchFilesByContentRequest.new(repository: @gitaly_repo, ref: ref, query: query) - response = GitalyClient.call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) + response = gitaly_client_call(@storage, :repository_service, :search_files_by_content, request, timeout: GitalyClient.default_timeout) search_results_from_response(response, options) end - def search_files_by_regexp(ref, filter) - request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter) - GitalyClient.call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) + def search_files_by_regexp(ref, filter, limit: 0, offset: 0) + request = Gitaly::SearchFilesByNameRequest.new(repository: @gitaly_repo, ref: ref, query: '.', filter: filter, limit: limit, offset: offset) + gitaly_client_call(@storage, :repository_service, :search_files_by_name, request, timeout: GitalyClient.fast_timeout).flat_map(&:files) end def disconnect_alternates @@ -324,19 +327,19 @@ module Gitlab repository: @gitaly_repo ) - GitalyClient.call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :object_pool_service, :disconnect_git_alternates, request, timeout: GitalyClient.long_timeout) end def rename(relative_path) request = Gitaly::RenameRepositoryRequest.new(repository: @gitaly_repo, relative_path: relative_path) - GitalyClient.call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) + gitaly_client_call(@storage, :repository_service, :rename_repository, request, timeout: GitalyClient.fast_timeout) end def remove request = Gitaly::RemoveRepositoryRequest.new(repository: @gitaly_repo) - GitalyClient.call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) + gitaly_client_call(@storage, :repository_service, :remove_repository, request, timeout: GitalyClient.long_timeout) end def replicate(source_repository) @@ -345,7 +348,7 @@ module Gitlab source: source_repository.gitaly_repository ) - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, :replicate_repository, @@ -371,11 +374,11 @@ module Gitlab current_match << message.match_data - if message.end_of_match - matches << current_match - current_match = +"" - matches_count += 1 - end + next unless message.end_of_match + + matches << current_match + current_match = +"" + matches_count += 1 end matches @@ -383,7 +386,7 @@ module Gitlab def gitaly_fetch_stream_to_file(save_path, rpc_name, request_class, timeout) request = request_class.new(repository: @gitaly_repo) - response = GitalyClient.call( + response = gitaly_client_call( @storage, :repository_service, rpc_name, @@ -416,7 +419,7 @@ module Gitlab end end - GitalyClient.call( + gitaly_client_call( @storage, :repository_service, rpc_name, diff --git a/lib/gitlab/gitaly_client/with_feature_flag_actors.rb b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb new file mode 100644 index 00000000000..92fc524b724 --- /dev/null +++ b/lib/gitlab/gitaly_client/with_feature_flag_actors.rb @@ -0,0 +1,103 @@ +# frozen_string_literal: true + +module Gitlab + module GitalyClient + # This module is responsible for collecting feature flag actors in Gitaly Client. Unlike normal feature flags used + # in Gitlab development, feature flags passed to Gitaly are pre-evaluated at Rails side before being passed to + # Gitaly. As a result, we need to collect all possible actors for the evaluation before issue any RPC. At this + # layer, the only parameter we have is raw repository. We need to infer other actors from the repository. Adding + # extra SQL queries before any RPC are not good for the performance. We applied some quirky optimizations here to + # avoid issuing SQL queries. However, in some less common code paths, a couple of queries are expected. + module WithFeatureFlagActors + include Gitlab::Utils::StrongMemoize + + attr_accessor :repository_actor + + # gitaly_client_call performs Gitaly calls including collected feature flag actors. The actors are retrieved + # from repository actor and memoized. The service must set `self.repository_actor = a_repository` beforehand. + def gitaly_client_call(*args, **kargs) + return GitalyClient.call(*args, **kargs) unless actors_aware_gitaly_calls? + + unless repository_actor + Gitlab::ErrorTracking.track_and_raise_for_dev_exception( + Feature::InvalidFeatureFlagError.new("gitaly_client_call called without setting repository_actor") + ) + end + + GitalyClient.with_feature_flag_actors( + repository: repository_actor, + user: user_actor, + project: project_actor, + group: group_actor + ) do + GitalyClient.call(*args, **kargs) + end + end + + # gitaly_feature_flag_actors returns a hash of actors implied from input repository. If actors_aware_gitaly_calls + # flag is not on, this method returns an empty hash. + def gitaly_feature_flag_actors(repository) + return {} unless actors_aware_gitaly_calls? + + container = find_repository_container(repository) + { + repository: repository, + user: Feature::Gitaly.user_actor, + project: Feature::Gitaly.project_actor(container), + group: Feature::Gitaly.group_actor(container) + } + end + + # Use actor here means the user who originally perform the action. It is collected from ApplicationContext. As + # this information is widely propagated in all entry points, User actor should be available everywhere, even in + # background jobs. + def user_actor + strong_memoize(:user_actor) do + Feature::Gitaly.user_actor + end + end + + # TODO: replace this project actor by Repo actor + def project_actor + strong_memoize(:project_actor) do + Feature::Gitaly.project_actor(repository_container) + end + end + + def group_actor + strong_memoize(:group_actor) do + Feature::Gitaly.group_actor(repository_container) + end + end + + private + + def repository_container + strong_memoize(:repository_container) do + find_repository_container(repository_actor) + end + end + + def find_repository_container(repository) + return if repository&.gl_repository.blank? + + if repository.container.nil? + begin + identifier = Gitlab::GlRepository::Identifier.parse(repository.gl_repository) + identifier.container + rescue Gitlab::GlRepository::Identifier::InvalidIdentifier + nil + end + else + repository.container + end + end + + def actors_aware_gitaly_calls? + Feature.enabled?(:actors_aware_gitaly_calls) + end + end + end +end + +Gitlab::GitalyClient::WithFeatureFlagActors.prepend_mod_with('Gitlab::GitalyClient::WithFeatureFlagActors') diff --git a/lib/gitlab/github_import/client.rb b/lib/gitlab/github_import/client.rb index 0f89a7b6575..d6060141bce 100644 --- a/lib/gitlab/github_import/client.rb +++ b/lib/gitlab/github_import/client.rb @@ -76,6 +76,10 @@ module Gitlab each_object(:pull_request_reviews, repo_name, iid) end + def pull_request_review_requests(repo_name, iid) + with_rate_limit { octokit.pull_request_review_requests(repo_name, iid).to_h } + end + def repos(options = {}) octokit.repos(nil, options).map(&:to_h) end diff --git a/lib/gitlab/github_import/importer/events/changed_assignee.rb b/lib/gitlab/github_import/importer/events/changed_assignee.rb index b75d41f40de..bcf9cd94ad9 100644 --- a/lib/gitlab/github_import/importer/events/changed_assignee.rb +++ b/lib/gitlab/github_import/importer/events/changed_assignee.rb @@ -39,12 +39,10 @@ module Gitlab def parse_body(issue_event, assignee_id) assignee = User.find(assignee_id).to_reference - Gitlab::I18n.with_default_locale do - if issue_event.event == "unassigned" - "unassigned #{assignee}" - else - "assigned to #{assignee}" - end + if issue_event.event == 'unassigned' + "#{SystemNotes::IssuablesService.issuable_events[:unassigned]} #{assignee}" + else + "#{SystemNotes::IssuablesService.issuable_events[:assigned]} #{assignee}" end end end diff --git a/lib/gitlab/github_import/importer/events/changed_label.rb b/lib/gitlab/github_import/importer/events/changed_label.rb index 83130d18db9..553ef0886e8 100644 --- a/lib/gitlab/github_import/importer/events/changed_label.rb +++ b/lib/gitlab/github_import/importer/events/changed_label.rb @@ -13,6 +13,7 @@ module Gitlab def create_event(issue_event) attrs = { + importing: true, user_id: author_id(issue_event), label_id: label_finder.id_for(issue_event.label_title), action: action(issue_event.event), diff --git a/lib/gitlab/github_import/importer/protected_branch_importer.rb b/lib/gitlab/github_import/importer/protected_branch_importer.rb index 21075e21e1d..801a0840c52 100644 --- a/lib/gitlab/github_import/importer/protected_branch_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branch_importer.rb @@ -37,18 +37,36 @@ module Gitlab name: protected_branch.id, push_access_levels_attributes: [{ access_level: push_access_level }], merge_access_levels_attributes: [{ access_level: merge_access_level }], - allow_force_push: allow_force_push? + allow_force_push: allow_force_push?, + code_owner_approval_required: code_owner_approval_required? } end def allow_force_push? - if ProtectedBranch.protected?(project, protected_branch.id) - ProtectedBranch.allow_force_push?(project, protected_branch.id) && protected_branch.allow_force_pushes + return false unless protected_branch.allow_force_pushes + + if protected_on_gitlab? + ProtectedBranch.allow_force_push?(project, protected_branch.id) + elsif default_branch? + !default_branch_protection.any? else - protected_branch.allow_force_pushes + true end end + def code_owner_approval_required? + return false unless project.licensed_feature_available?(:code_owner_approval_required) + + return protected_branch.require_code_owner_reviews unless protected_on_gitlab? + + # Gets the strictest require_code_owner rule between GitHub and GitLab + protected_branch.require_code_owner_reviews || + ProtectedBranch.branch_requires_code_owner_approval?( + project, + protected_branch.id + ) + end + def default_branch? protected_branch.id == project.default_branch end diff --git a/lib/gitlab/github_import/importer/protected_branches_importer.rb b/lib/gitlab/github_import/importer/protected_branches_importer.rb index 4372477f55d..ff425528aec 100644 --- a/lib/gitlab/github_import/importer/protected_branches_importer.rb +++ b/lib/gitlab/github_import/importer/protected_branches_importer.rb @@ -13,13 +13,15 @@ module Gitlab protected_branches = client.branches(repo).select { |branch| branch.dig(:protection, :enabled) } protected_branches.each do |protected_branch| + next if already_imported?(protected_branch) + object = client.branch_protection(repo, protected_branch[:name]) - next if object.nil? || already_imported?(object) + next if object.nil? yield object Gitlab::GithubImport::ObjectCounter.increment(project, object_type, :fetched) - mark_as_imported(object) + mark_as_imported(protected_branch) end end diff --git a/lib/gitlab/github_import/importer/pull_request_review_importer.rb b/lib/gitlab/github_import/importer/pull_request_review_importer.rb index dd5b7c93ced..b11af90aa6f 100644 --- a/lib/gitlab/github_import/importer/pull_request_review_importer.rb +++ b/lib/gitlab/github_import/importer/pull_request_review_importer.rb @@ -18,6 +18,7 @@ module Gitlab if gitlab_user_id add_review_note!(gitlab_user_id) add_approval!(gitlab_user_id) + add_reviewer!(gitlab_user_id) else add_complementary_review_note!(project.creator_id) end @@ -95,6 +96,24 @@ module Gitlab end end + def add_reviewer!(user_id) + return if review_re_requested?(user_id) + + ::MergeRequestReviewer.create!( + merge_request_id: merge_request.id, + user_id: user_id, + state: ::MergeRequestReviewer.states['reviewed'], + created_at: submitted_at + ) + end + + # rubocop:disable CodeReuse/ActiveRecord + def review_re_requested?(user_id) + # records that were imported on previous stage with "unreviewed" status + MergeRequestReviewer.where(merge_request_id: merge_request.id, user_id: user_id).exists? + end + # rubocop:enable CodeReuse/ActiveRecord + def add_approval_system_note!(user_id) attributes = note_attributes( user_id, diff --git a/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb new file mode 100644 index 00000000000..bb51d856d9b --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/review_request_importer.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewRequestImporter + def initialize(review_request, project, client) + @review_request = review_request + @user_finder = UserFinder.new(project, client) + @issue_finder = IssuableFinder.new(project, client) + end + + def execute + MergeRequestReviewer.bulk_insert!(build_reviewers) + end + + private + + attr_reader :review_request, :user_finder + + def build_reviewers + reviewer_ids = review_request.users.map { |user| user_finder.user_id_for(user) }.compact + + reviewer_ids.map do |reviewer_id| + MergeRequestReviewer.new( + merge_request_id: review_request.merge_request_id, + user_id: reviewer_id, + state: MergeRequestReviewer.states['unreviewed'], + created_at: Time.zone.now + ) + end + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb new file mode 100644 index 00000000000..c5d8da3be1c --- /dev/null +++ b/lib/gitlab/github_import/importer/pull_requests/review_requests_importer.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Importer + module PullRequests + class ReviewRequestsImporter + include ParallelScheduling + + BATCH_SIZE = 100 + + private + + def each_object_to_import(&block) + merge_request_collection.each_batch(of: BATCH_SIZE, column: :iid) do |batch| + batch.each do |merge_request| + repo = project.import_source + + review_requests = client.pull_request_review_requests(repo, merge_request.iid) + review_requests[:merge_request_id] = merge_request.id + yield review_requests + + mark_merge_request_imported(merge_request) + end + end + end + + def importer_class + ReviewRequestImporter + end + + def representation_class + Gitlab::GithubImport::Representation::PullRequests::ReviewRequests + end + + def sidekiq_worker_class + Gitlab::GithubImport::PullRequests::ImportReviewRequestWorker + end + + def collection_method + :pull_request_review_requests + end + + # rubocop:disable CodeReuse/ActiveRecord + def merge_request_collection + project.merge_requests + .where.not(iid: already_imported_merge_requests) + .select(:id, :iid) + end + # rubocop:enable CodeReuse/ActiveRecord + + def merge_request_imported_cache_key + "github-importer/pull_requests/#{collection_method}/already-imported/#{project.id}" + end + + def already_imported_merge_requests + Gitlab::Cache::Import::Caching.values_from_set(merge_request_imported_cache_key) + end + + def mark_merge_request_imported(merge_request) + Gitlab::Cache::Import::Caching.set_add( + merge_request_imported_cache_key, + merge_request.iid + ) + end + end + end + end + end +end diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 16541c90002..62863ba67fd 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -38,7 +38,7 @@ module Gitlab # deliberate. If we were to update this column after the fetch we may # miss out on changes pushed during the fetch or between the fetch and # updating the timestamp. - project.touch(:last_repository_updated_at) # rubocop: disable Rails/SkipsModelValidations + project.touch(:last_repository_updated_at) project.repository.fetch_remote(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true) diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 708768a60cf..d7fe01e90f8 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -80,7 +80,7 @@ module Gitlab end def update_clone_time - project.touch(:last_repository_updated_at) # rubocop: disable Rails/SkipsModelValidations + project.touch(:last_repository_updated_at) end private diff --git a/lib/gitlab/github_import/representation/protected_branch.rb b/lib/gitlab/github_import/representation/protected_branch.rb index 07a607ae70d..d2a52b64bbf 100644 --- a/lib/gitlab/github_import/representation/protected_branch.rb +++ b/lib/gitlab/github_import/representation/protected_branch.rb @@ -10,7 +10,7 @@ module Gitlab attr_reader :attributes expose_attribute :id, :allow_force_pushes, :required_conversation_resolution, :required_signatures, - :required_pull_request_reviews + :required_pull_request_reviews, :require_code_owner_reviews # Builds a Branch Protection info from a GitHub API response. # Resource structure details: @@ -24,7 +24,9 @@ module Gitlab allow_force_pushes: branch_protection.dig(:allow_force_pushes, :enabled), required_conversation_resolution: branch_protection.dig(:required_conversation_resolution, :enabled), required_signatures: branch_protection.dig(:required_signatures, :enabled), - required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present? + required_pull_request_reviews: branch_protection[:required_pull_request_reviews].present?, + require_code_owner_reviews: branch_protection.dig(:required_pull_request_reviews, + :require_code_owner_reviews).present? } new(hash) diff --git a/lib/gitlab/github_import/representation/pull_requests/review_requests.rb b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb new file mode 100644 index 00000000000..692004c4460 --- /dev/null +++ b/lib/gitlab/github_import/representation/pull_requests/review_requests.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module GithubImport + module Representation + module PullRequests + class ReviewRequests + include ToHash + include ExposeAttribute + + attr_reader :attributes + + expose_attribute :merge_request_id, :users + + class << self + # Builds a list of requested reviewers from a GitHub API response. + # + # review_requests - An instance of `Hash` containing the review requests details. + def from_api_response(review_requests, _additional_data = {}) + review_requests = Representation.symbolize_hash(review_requests) + users = review_requests[:users].map do |user_data| + Representation::User.from_api_response(user_data) + end + + new( + merge_request_id: review_requests[:merge_request_id], + users: users + ) + end + alias_method :from_json_hash, :from_api_response + end + + # attributes - A Hash containing the review details. The keys of this + # Hash (and any nested hashes) must be symbols. + def initialize(attributes) + @attributes = attributes + end + + def github_identifiers + { merge_request_id: merge_request_id } + end + end + end + end + end +end diff --git a/lib/gitlab/gon_helper.rb b/lib/gitlab/gon_helper.rb index bdb7484f3d6..ecb57bfc1a2 100644 --- a/lib/gitlab/gon_helper.rb +++ b/lib/gitlab/gon_helper.rb @@ -18,8 +18,16 @@ module Gitlab gon.markdown_automatic_lists = current_user&.markdown_automatic_lists if Gitlab.config.sentry.enabled - gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn - gon.sentry_environment = Gitlab.config.sentry.environment + gon.sentry_dsn = Gitlab.config.sentry.clientside_dsn + gon.sentry_environment = Gitlab.config.sentry.environment + end + + # Support for Sentry setup via configuration files will be removed in 16.0 + # in favor of Gitlab::CurrentSettings. + if Feature.enabled?(:enable_new_sentry_clientside_integration, + current_user) && Gitlab::CurrentSettings.sentry_enabled + gon.sentry_dsn = Gitlab::CurrentSettings.sentry_clientside_dsn + gon.sentry_environment = Gitlab::CurrentSettings.sentry_environment end gon.recaptcha_api_server_url = ::Recaptcha.configuration.api_server_url @@ -58,6 +66,7 @@ module Gitlab push_frontend_feature_flag(:new_header_search) push_frontend_feature_flag(:source_editor_toolbar) push_frontend_feature_flag(:integration_slack_app_notifications) + push_frontend_feature_flag(:vue_group_select) end # Exposes the state of a feature flag to the frontend code. diff --git a/lib/gitlab/grape_logging/loggers/filter_parameters.rb b/lib/gitlab/grape_logging/loggers/filter_parameters.rb new file mode 100644 index 00000000000..ae9df203544 --- /dev/null +++ b/lib/gitlab/grape_logging/loggers/filter_parameters.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +module Gitlab + module GrapeLogging + module Loggers + # In the CI variables APIs, the POST or PUT parameters will always be + # literally 'key' and 'value'. Rails' default filters_parameters will + # always incorrectly mask the value of param 'key' when it should mask the + # value of the param 'value'. + # See: https://gitlab.com/gitlab-org/gitlab/-/issues/353857 + class FilterParameters < ::GrapeLogging::Loggers::FilterParameters + private + + def safe_parameters(request) + loggable_params = super + settings = request.env[Grape::Env::API_ENDPOINT]&.route&.settings + + return loggable_params unless settings&.key?(:log_safety) + + settings[:log_safety][:safe].each do |key| + loggable_params[key] = request.params[key] if loggable_params.key?(key) + end + + settings[:log_safety][:unsafe].each do |key| + loggable_params[key] = @replacement if loggable_params.key?(key) + end + + loggable_params + end + end + end + end +end diff --git a/lib/gitlab/highlight.rb b/lib/gitlab/highlight.rb index b71abe5c052..1a85c57e6b1 100644 --- a/lib/gitlab/highlight.rb +++ b/lib/gitlab/highlight.rb @@ -2,9 +2,9 @@ module Gitlab class Highlight - def self.highlight(blob_name, blob_content, language: nil, plain: false) + def self.highlight(blob_name, blob_content, language: nil, plain: false, context: {}) new(blob_name, blob_content, language: language) - .highlight(blob_content, continue: false, plain: plain) + .highlight(blob_content, continue: false, plain: plain, context: context) end def self.too_large?(size) diff --git a/lib/gitlab/hook_data/merge_request_builder.rb b/lib/gitlab/hook_data/merge_request_builder.rb index 65c623c5d7d..96128f432c5 100644 --- a/lib/gitlab/hook_data/merge_request_builder.rb +++ b/lib/gitlab/hook_data/merge_request_builder.rb @@ -66,12 +66,19 @@ module Gitlab labels: merge_request.labels_hook_attrs, state: merge_request.state, # This key is deprecated blocking_discussions_resolved: merge_request.mergeable_discussions_state?, - first_contribution: merge_request.first_contribution? + first_contribution: merge_request.first_contribution?, + detailed_merge_status: detailed_merge_status } merge_request.attributes.with_indifferent_access.slice(*self.class.safe_hook_attributes) .merge!(attrs) end + + private + + def detailed_merge_status + ::MergeRequests::Mergeability::DetailedMergeStatusService.new(merge_request: merge_request).execute.to_s + end end end end diff --git a/lib/gitlab/i18n.rb b/lib/gitlab/i18n.rb index a2d06b7f5b3..a42cac61a55 100644 --- a/lib/gitlab/i18n.rb +++ b/lib/gitlab/i18n.rb @@ -44,30 +44,30 @@ module Gitlab TRANSLATION_LEVELS = { 'bg' => 0, 'cs_CZ' => 0, - 'da_DK' => 37, + 'da_DK' => 36, 'de' => 17, 'en' => 100, 'eo' => 0, - 'es' => 36, + 'es' => 35, 'fil_PH' => 0, - 'fr' => 72, + 'fr' => 85, 'gl_ES' => 0, 'id_ID' => 0, 'it' => 1, - 'ja' => 31, - 'ko' => 20, + 'ja' => 30, + 'ko' => 21, 'nb_NO' => 25, 'nl_NL' => 0, 'pl_PL' => 3, - 'pt_BR' => 57, - 'ro_RO' => 99, - 'ru' => 26, + 'pt_BR' => 58, + 'ro_RO' => 98, + 'ru' => 25, 'si_LK' => 11, 'tr_TR' => 11, - 'uk' => 49, + 'uk' => 52, 'zh_CN' => 98, 'zh_HK' => 1, - 'zh_TW' => 99 + 'zh_TW' => 100 }.freeze private_constant :TRANSLATION_LEVELS diff --git a/lib/gitlab/identifier.rb b/lib/gitlab/identifier.rb index d5f94ad04f1..08d44184bb6 100644 --- a/lib/gitlab/identifier.rb +++ b/lib/gitlab/identifier.rb @@ -5,10 +5,11 @@ module Gitlab module Identifier def identify(identifier) - if identifier =~ /\Auser-\d+\Z/ + case identifier + when /\Auser-\d+\Z/ # git push over http identify_using_user(identifier) - elsif identifier =~ /\Akey-\d+\Z/ + when /\Akey-\d+\Z/ # git push over ssh identify_using_ssh_key(identifier) end diff --git a/lib/gitlab/import_export/attributes_permitter.rb b/lib/gitlab/import_export/attributes_permitter.rb index f6f65f85599..8c7a6c13246 100644 --- a/lib/gitlab/import_export/attributes_permitter.rb +++ b/lib/gitlab/import_export/attributes_permitter.rb @@ -85,11 +85,11 @@ module Gitlab while stack.any? model_name, relations = stack.pop - if relations.is_a?(Hash) - add_permitted_attributes(model_name, relations.keys) + next unless relations.is_a?(Hash) - stack.concat(relations.to_a) - end + add_permitted_attributes(model_name, relations.keys) + + stack.concat(relations.to_a) end @permitted_attributes diff --git a/lib/gitlab/import_export/base/relation_object_saver.rb b/lib/gitlab/import_export/base/relation_object_saver.rb index 3c473449ec0..ed3858d0bf4 100644 --- a/lib/gitlab/import_export/base/relation_object_saver.rb +++ b/lib/gitlab/import_export/base/relation_object_saver.rb @@ -81,11 +81,11 @@ module Gitlab subrelation = relation_object.public_send(definition) association = relation_object.class.reflect_on_association(definition) - if association&.collection? && subrelation.size > MIN_RECORDS_SIZE - collection_subrelations[definition] = subrelation.records + next unless association&.collection? && subrelation.size > MIN_RECORDS_SIZE - subrelation.clear - end + collection_subrelations[definition] = subrelation.records + + subrelation.clear end end end diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index c98dcf7b848..aa66fe8a5ae 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -87,7 +87,6 @@ module Gitlab def validate_archive_path Gitlab::Utils.check_path_traversal!(@archive_path) - raise(ServiceError, 'Archive path is not a string') unless @archive_path.is_a?(String) raise(ServiceError, 'Archive path is a symlink') if File.lstat(@archive_path).symlink? raise(ServiceError, 'Archive path is not a file') unless File.file?(@archive_path) end diff --git a/lib/gitlab/import_export/project/exported_relations_merger.rb b/lib/gitlab/import_export/project/exported_relations_merger.rb new file mode 100644 index 00000000000..dda3d00d608 --- /dev/null +++ b/lib/gitlab/import_export/project/exported_relations_merger.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module Project + class ExportedRelationsMerger + include Gitlab::ImportExport::CommandLineUtil + + def initialize(export_job:, shared:) + @export_job = export_job + @shared = shared + end + + def save + Dir.mktmpdir do |dirpath| + export_job.relation_exports.each do |relation_export| + relation = relation_export.relation + upload = relation_export.upload + filename = upload.export_file.filename + + tar_gz_full_path = File.join(dirpath, filename) + decompress_path = File.join(dirpath, relation) + Gitlab::Utils.check_path_traversal!(tar_gz_full_path) + Gitlab::Utils.check_path_traversal!(decompress_path) + + # Download tar.gz + download_or_copy_upload( + upload.export_file, tar_gz_full_path, size_limit: relation_export.upload.export_file.size + ) + + # Decompress tar.gz + mkdir_p(decompress_path) + untar_zxf(dir: decompress_path, archive: tar_gz_full_path) + File.delete(tar_gz_full_path) + + # Merge decompressed files into export_path + RecursiveMergeFolders.merge(decompress_path, shared.export_path) + FileUtils.rm_r(decompress_path) + rescue StandardError => e + shared.error(e) + false + end + end + + shared.errors.empty? + end + + private + + attr_reader :shared, :export_job + + delegate :project, to: :export_job + end + end + end +end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index fb44aaf094e..2d9c8d1108e 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -302,6 +302,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level prometheus_metrics: - :created_at - :updated_at @@ -585,7 +586,7 @@ included_attributes: - :target_sha pipeline_metadata: - :project_id - - :title + - :name stages: - :name - :status @@ -717,6 +718,7 @@ included_attributes: - :environments_access_level - :feature_flags_access_level - :releases_access_level + - :infrastructure_access_level - :allow_merge_on_skipped_pipeline - :auto_devops_deploy_strategy - :auto_devops_enabled diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index 8e91adac196..967239e17c1 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,7 @@ module Gitlab project, reader.project_tree, json_writer, - exportable_path: 'project', + exportable_path: 'tree/project', current_user: nil ) end diff --git a/lib/gitlab/import_export/recursive_merge_folders.rb b/lib/gitlab/import_export/recursive_merge_folders.rb new file mode 100644 index 00000000000..982358699bd --- /dev/null +++ b/lib/gitlab/import_export/recursive_merge_folders.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +# +# This class is used by Import/Export to move files and folders from a source folders into a target folders +# that can already have the same folders in it, resolving in a merged folder. +# +# Example: +# +# source path +# |-- tree +# | |-- project +# | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | |-- folder2 +# | | |-- image2.png +# +# target path +# |-- tree +# | |-- project +# | |-- issues.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image3.png +# | |-- folder3 +# | | |-- image4.png +# +# target path after merge +# |-- tree +# | |-- project +# | | |-- issues.ndjson +# | | |-- labels.ndjson +# |-- uploads +# | |-- folder1 +# | | |-- image1.png +# | | |-- image3.png +# | |-- folder2 +# | | |-- image2.png +# | |-- folder3 +# | | |-- image4.png + +module Gitlab + module ImportExport + class RecursiveMergeFolders + DEFAULT_DIR_MODE = 0o700 + + def self.merge(source_path, target_path) + Gitlab::Utils.check_path_traversal!(source_path) + Gitlab::Utils.check_path_traversal!(target_path) + Gitlab::Utils.check_allowed_absolute_path!(source_path, [Dir.tmpdir]) + + recursive_merge(source_path, target_path) + end + + def self.recursive_merge(source_path, target_path) + Dir.children(source_path).each do |child| + source_child = File.join(source_path, child) + target_child = File.join(target_path, child) + + next if File.lstat(source_child).symlink? + + if File.directory?(source_child) + FileUtils.mkdir_p(target_child, mode: DEFAULT_DIR_MODE) unless File.exist?(target_child) + recursive_merge(source_child, target_child) + else + FileUtils.mv(source_child, target_child) + end + end + end + + private_class_method :recursive_merge + end + end +end diff --git a/lib/gitlab/incoming_email.rb b/lib/gitlab/incoming_email.rb index d55906083ff..d34c19bc9fc 100644 --- a/lib/gitlab/incoming_email.rb +++ b/lib/gitlab/incoming_email.rb @@ -2,30 +2,11 @@ module Gitlab module IncomingEmail - UNSUBSCRIBE_SUFFIX = '-unsubscribe' - UNSUBSCRIBE_SUFFIX_LEGACY = '+unsubscribe' - WILDCARD_PLACEHOLDER = '%{key}' - class << self - def enabled? - config.enabled && config.address.present? - end + include Gitlab::Email::Common - def supports_wildcard? - config.address.present? && config.address.include?(WILDCARD_PLACEHOLDER) - end - - def supports_issue_creation? - enabled? && supports_wildcard? - end - - def reply_address(key) - config.address.sub(WILDCARD_PLACEHOLDER, key) - end - - # example: incoming+1234567890abcdef1234567890abcdef-unsubscribe@incoming.gitlab.com - def unsubscribe_address(key) - config.address.sub(WILDCARD_PLACEHOLDER, "#{key}#{UNSUBSCRIBE_SUFFIX}") + def config + incoming_email_config end def key_from_address(address, wildcard_address: nil) @@ -39,21 +20,6 @@ module Gitlab match[1] end - def key_from_fallback_message_id(mail_id) - message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - - mail_id[message_id_regexp, 1] - end - - def scan_fallback_references(references) - # It's looking for each <...> - references.scan(/(?!<)[^<>]+(?=>)/) - end - - def config - Gitlab.config.incoming_email - end - private def address_regex(wildcard_address) diff --git a/lib/gitlab/instrumentation/redis_base.rb b/lib/gitlab/instrumentation/redis_base.rb index 0bd10597f24..268c6cdf459 100644 --- a/lib/gitlab/instrumentation/redis_base.rb +++ b/lib/gitlab/instrumentation/redis_base.rb @@ -66,8 +66,8 @@ module Gitlab query_time.round(::Gitlab::InstrumentationHelper::DURATION_PRECISION) end - def redis_cluster_validate!(command) - ::Gitlab::Instrumentation::RedisClusterValidator.validate!(command) if @redis_cluster_validation + def redis_cluster_validate!(commands) + ::Gitlab::Instrumentation::RedisClusterValidator.validate!(commands) if @redis_cluster_validation end def enable_redis_cluster_validation diff --git a/lib/gitlab/instrumentation/redis_cluster_validator.rb b/lib/gitlab/instrumentation/redis_cluster_validator.rb index 005751fb0db..36d3e088956 100644 --- a/lib/gitlab/instrumentation/redis_cluster_validator.rb +++ b/lib/gitlab/instrumentation/redis_cluster_validator.rb @@ -10,57 +10,189 @@ module Gitlab # # Gitlab::Redis::Cache # .with { |redis| redis.call('COMMAND') } - # .select { |command| command[3] != command[4] } - # .map { |command| [command[0].upcase, { first: command[3], last: command[4], step: command[5] }] } + # .select { |cmd| cmd[3] != 0 } + # .map { |cmd| [ + # cmd[0].upcase, + # { first: cmd[3], last: cmd[4], step: cmd[5], single_key: cmd[3] == cmd[4] } + # ] + # } # .sort_by(&:first) # .to_h - # - MULTI_KEY_COMMANDS = { - "BITOP" => { first: 2, last: -1, step: 1 }, - "BLPOP" => { first: 1, last: -2, step: 1 }, - "BRPOP" => { first: 1, last: -2, step: 1 }, - "BRPOPLPUSH" => { first: 1, last: 2, step: 1 }, - "BZPOPMAX" => { first: 1, last: -2, step: 1 }, - "BZPOPMIN" => { first: 1, last: -2, step: 1 }, - "DEL" => { first: 1, last: -1, step: 1 }, - "EXISTS" => { first: 1, last: -1, step: 1 }, - "MGET" => { first: 1, last: -1, step: 1 }, - "MSET" => { first: 1, last: -1, step: 2 }, - "MSETNX" => { first: 1, last: -1, step: 2 }, - "PFCOUNT" => { first: 1, last: -1, step: 1 }, - "PFMERGE" => { first: 1, last: -1, step: 1 }, - "RENAME" => { first: 1, last: 2, step: 1 }, - "RENAMENX" => { first: 1, last: 2, step: 1 }, - "RPOPLPUSH" => { first: 1, last: 2, step: 1 }, - "SDIFF" => { first: 1, last: -1, step: 1 }, - "SDIFFSTORE" => { first: 1, last: -1, step: 1 }, - "SINTER" => { first: 1, last: -1, step: 1 }, - "SINTERSTORE" => { first: 1, last: -1, step: 1 }, - "SMOVE" => { first: 1, last: 2, step: 1 }, - "SUNION" => { first: 1, last: -1, step: 1 }, - "SUNIONSTORE" => { first: 1, last: -1, step: 1 }, - "UNLINK" => { first: 1, last: -1, step: 1 }, - "WATCH" => { first: 1, last: -1, step: 1 } + REDIS_COMMANDS = { + "APPEND" => { first: 1, last: 1, step: 1, single_key: true }, + "BITCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "BITFIELD" => { first: 1, last: 1, step: 1, single_key: true }, + "BITFIELD_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "BITOP" => { first: 2, last: -1, step: 1, single_key: false }, + "BITPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "BLMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "BLPOP" => { first: 1, last: -2, step: 1, single_key: false }, + "BRPOP" => { first: 1, last: -2, step: 1, single_key: false }, + "BRPOPLPUSH" => { first: 1, last: 2, step: 1, single_key: false }, + "BZPOPMAX" => { first: 1, last: -2, step: 1, single_key: false }, + "BZPOPMIN" => { first: 1, last: -2, step: 1, single_key: false }, + "COPY" => { first: 1, last: 2, step: 1, single_key: false }, + "DECR" => { first: 1, last: 1, step: 1, single_key: true }, + "DECRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "DEL" => { first: 1, last: -1, step: 1, single_key: false }, + "DUMP" => { first: 1, last: 1, step: 1, single_key: true }, + "EXISTS" => { first: 1, last: -1, step: 1, single_key: false }, + "EXPIRE" => { first: 1, last: 1, step: 1, single_key: true }, + "EXPIREAT" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOADD" => { first: 1, last: 1, step: 1, single_key: true }, + "GEODIST" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOHASH" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUS" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUSBYMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUSBYMEMBER_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "GEORADIUS_RO" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOSEARCH" => { first: 1, last: 1, step: 1, single_key: true }, + "GEOSEARCHSTORE" => { first: 1, last: 2, step: 1, single_key: false }, + "GET" => { first: 1, last: 1, step: 1, single_key: true }, + "GETBIT" => { first: 1, last: 1, step: 1, single_key: true }, + "GETDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "GETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "GETRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "GETSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "HEXISTS" => { first: 1, last: 1, step: 1, single_key: true }, + "HGET" => { first: 1, last: 1, step: 1, single_key: true }, + "HGETALL" => { first: 1, last: 1, step: 1, single_key: true }, + "HINCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "HINCRBYFLOAT" => { first: 1, last: 1, step: 1, single_key: true }, + "HKEYS" => { first: 1, last: 1, step: 1, single_key: true }, + "HLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "HMGET" => { first: 1, last: 1, step: 1, single_key: true }, + "HMSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HRANDFIELD" => { first: 1, last: 1, step: 1, single_key: true }, + "HSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "HSET" => { first: 1, last: 1, step: 1, single_key: true }, + "HSETNX" => { first: 1, last: 1, step: 1, single_key: true }, + "HSTRLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "HVALS" => { first: 1, last: 1, step: 1, single_key: true }, + "INCR" => { first: 1, last: 1, step: 1, single_key: true }, + "INCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "INCRBYFLOAT" => { first: 1, last: 1, step: 1, single_key: true }, + "LINDEX" => { first: 1, last: 1, step: 1, single_key: true }, + "LINSERT" => { first: 1, last: 1, step: 1, single_key: true }, + "LLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "LMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "LPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "LPOS" => { first: 1, last: 1, step: 1, single_key: true }, + "LPUSH" => { first: 1, last: 1, step: 1, single_key: true }, + "LPUSHX" => { first: 1, last: 1, step: 1, single_key: true }, + "LRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "LREM" => { first: 1, last: 1, step: 1, single_key: true }, + "LSET" => { first: 1, last: 1, step: 1, single_key: true }, + "LTRIM" => { first: 1, last: 1, step: 1, single_key: true }, + "MGET" => { first: 1, last: -1, step: 1, single_key: false }, + "MIGRATE" => { first: 3, last: 3, step: 1, single_key: true }, + "MOVE" => { first: 1, last: 1, step: 1, single_key: true }, + "MSET" => { first: 1, last: -1, step: 2, single_key: false }, + "MSETNX" => { first: 1, last: -1, step: 2, single_key: false }, + "OBJECT" => { first: 2, last: 2, step: 1, single_key: true }, + "PERSIST" => { first: 1, last: 1, step: 1, single_key: true }, + "PEXPIRE" => { first: 1, last: 1, step: 1, single_key: true }, + "PEXPIREAT" => { first: 1, last: 1, step: 1, single_key: true }, + "PFADD" => { first: 1, last: 1, step: 1, single_key: true }, + "PFCOUNT" => { first: 1, last: -1, step: 1, single_key: false }, + "PFDEBUG" => { first: 2, last: 2, step: 1, single_key: true }, + "PFMERGE" => { first: 1, last: -1, step: 1, single_key: false }, + "PSETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "PTTL" => { first: 1, last: 1, step: 1, single_key: true }, + "RENAME" => { first: 1, last: 2, step: 1, single_key: false }, + "RENAMENX" => { first: 1, last: 2, step: 1, single_key: false }, + "RESTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "RESTORE-ASKING" => { first: 1, last: 1, step: 1, single_key: true }, + "RPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "RPOPLPUSH" => { first: 1, last: 2, step: 1, single_key: false }, + "RPUSH" => { first: 1, last: 1, step: 1, single_key: true }, + "RPUSHX" => { first: 1, last: 1, step: 1, single_key: true }, + "SADD" => { first: 1, last: 1, step: 1, single_key: true }, + "SCARD" => { first: 1, last: 1, step: 1, single_key: true }, + "SDIFF" => { first: 1, last: -1, step: 1, single_key: false }, + "SDIFFSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "SET" => { first: 1, last: 1, step: 1, single_key: true }, + "SETBIT" => { first: 1, last: 1, step: 1, single_key: true }, + "SETEX" => { first: 1, last: 1, step: 1, single_key: true }, + "SETNX" => { first: 1, last: 1, step: 1, single_key: true }, + "SETRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "SINTER" => { first: 1, last: -1, step: 1, single_key: false }, + "SINTERSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "SISMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SMEMBERS" => { first: 1, last: 1, step: 1, single_key: true }, + "SMISMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SMOVE" => { first: 1, last: 2, step: 1, single_key: false }, + "SORT" => { first: 1, last: 1, step: 1, single_key: true }, + "SPOP" => { first: 1, last: 1, step: 1, single_key: true }, + "SRANDMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "SREM" => { first: 1, last: 1, step: 1, single_key: true }, + "SSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "STRLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "SUBSTR" => { first: 1, last: 1, step: 1, single_key: true }, + "SUNION" => { first: 1, last: -1, step: 1, single_key: false }, + "SUNIONSTORE" => { first: 1, last: -1, step: 1, single_key: false }, + "TOUCH" => { first: 1, last: -1, step: 1, single_key: false }, + "TTL" => { first: 1, last: 1, step: 1, single_key: true }, + "TYPE" => { first: 1, last: 1, step: 1, single_key: true }, + "UNLINK" => { first: 1, last: -1, step: 1, single_key: false }, + "WATCH" => { first: 1, last: -1, step: 1, single_key: false }, + "XACK" => { first: 1, last: 1, step: 1, single_key: true }, + "XADD" => { first: 1, last: 1, step: 1, single_key: true }, + "XAUTOCLAIM" => { first: 1, last: 1, step: 1, single_key: true }, + "XCLAIM" => { first: 1, last: 1, step: 1, single_key: true }, + "XDEL" => { first: 1, last: 1, step: 1, single_key: true }, + "XGROUP" => { first: 2, last: 2, step: 1, single_key: true }, + "XINFO" => { first: 2, last: 2, step: 1, single_key: true }, + "XLEN" => { first: 1, last: 1, step: 1, single_key: true }, + "XPENDING" => { first: 1, last: 1, step: 1, single_key: true }, + "XRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "XREVRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "XSETID" => { first: 1, last: 1, step: 1, single_key: true }, + "XTRIM" => { first: 1, last: 1, step: 1, single_key: true }, + "ZADD" => { first: 1, last: 1, step: 1, single_key: true }, + "ZCARD" => { first: 1, last: 1, step: 1, single_key: true }, + "ZCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "ZDIFFSTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZINCRBY" => { first: 1, last: 1, step: 1, single_key: true }, + "ZINTERSTORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZLEXCOUNT" => { first: 1, last: 1, step: 1, single_key: true }, + "ZMSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZPOPMAX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZPOPMIN" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANDMEMBER" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZRANGESTORE" => { first: 1, last: 2, step: 1, single_key: false }, + "ZRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREM" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREMRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGEBYLEX" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANGEBYSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZREVRANK" => { first: 1, last: 1, step: 1, single_key: true }, + "ZSCAN" => { first: 1, last: 1, step: 1, single_key: true }, + "ZSCORE" => { first: 1, last: 1, step: 1, single_key: true }, + "ZUNIONSTORE" => { first: 1, last: 1, step: 1, single_key: true } }.freeze CrossSlotError = Class.new(StandardError) class << self - def validate!(command) + def validate!(commands) return unless Rails.env.development? || Rails.env.test? return if allow_cross_slot_commands? + return if commands.empty? - command_name = command.first.to_s.upcase - argument_positions = MULTI_KEY_COMMANDS[command_name] - - return unless argument_positions - - arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] - - key_slots = arguments.each_slice(argument_positions[:step]).map do |args| - key_slot(args.first) - end + # early exit for single-command (non-pipelined) if it is a single-key-command + command_name = commands.size > 1 ? "PIPELINE/MULTI" : commands.first.first.to_s.upcase + return if commands.size == 1 && REDIS_COMMANDS.dig(command_name, :single_key) + key_slots = commands.map { |command| key_slots(command) }.flatten if key_slots.uniq.many? # rubocop: disable CodeReuse/ActiveRecord raise CrossSlotError, "Redis command #{command_name} arguments hash to different slots. See https://docs.gitlab.com/ee/development/redis.html#multi-key-commands" end @@ -78,6 +210,17 @@ module Gitlab private + def key_slots(command) + argument_positions = REDIS_COMMANDS[command.first.to_s.upcase] + + return [] unless argument_positions + + arguments = command.flatten[argument_positions[:first]..argument_positions[:last]] + arguments.each_slice(argument_positions[:step]).map do |args| + key_slot(args.first) + end + end + def allow_cross_slot_commands? Thread.current[:allow_cross_slot_commands].to_i > 0 end diff --git a/lib/gitlab/instrumentation/redis_interceptor.rb b/lib/gitlab/instrumentation/redis_interceptor.rb index 7e2acb91b94..f19279df2fe 100644 --- a/lib/gitlab/instrumentation/redis_interceptor.rb +++ b/lib/gitlab/instrumentation/redis_interceptor.rb @@ -5,14 +5,6 @@ module Gitlab module RedisInterceptor APDEX_EXCLUDE = %w[brpop blpop brpoplpush bzpopmin bzpopmax xread xreadgroup].freeze - class MysteryRedisDurationError < StandardError - attr_reader :backtrace - - def initialize(backtrace) - @backtrace = backtrace - end - end - def call(command) instrument_call([command]) do super @@ -41,8 +33,7 @@ module Gitlab def instrument_call(commands) start = Gitlab::Metrics::System.monotonic_time # must come first so that 'start' is always defined instrumentation_class.instance_count_request(commands.size) - - commands.each { |c| instrumentation_class.redis_cluster_validate!(c) } + instrumentation_class.redis_cluster_validate!(commands) yield rescue ::Redis::BaseError => ex diff --git a/lib/gitlab/issues/rebalancing/state.rb b/lib/gitlab/issues/rebalancing/state.rb index abb50281f7a..36346564b39 100644 --- a/lib/gitlab/issues/rebalancing/state.rb +++ b/lib/gitlab/issues/rebalancing/state.rb @@ -25,7 +25,7 @@ module Gitlab redis.multi do |multi| # we trigger re-balance for namespaces(groups) or specific user project value = "#{rebalanced_container_type}/#{rebalanced_container_id}" - multi.sadd(CONCURRENT_RUNNING_REBALANCES_KEY, value) + multi.sadd?(CONCURRENT_RUNNING_REBALANCES_KEY, value) multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) end end @@ -99,11 +99,13 @@ module Gitlab def refresh_keys_expiration with_redis do |redis| - redis.multi do |multi| - multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) - multi.expire(current_index_key, REDIS_EXPIRY_TIME) - multi.expire(current_project_key, REDIS_EXPIRY_TIME) - multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.expire(issue_ids_key, REDIS_EXPIRY_TIME) + multi.expire(current_index_key, REDIS_EXPIRY_TIME) + multi.expire(current_project_key, REDIS_EXPIRY_TIME) + multi.expire(CONCURRENT_RUNNING_REBALANCES_KEY, REDIS_EXPIRY_TIME) + end end end end @@ -112,12 +114,14 @@ module Gitlab value = "#{rebalanced_container_type}/#{rebalanced_container_id}" with_redis do |redis| - redis.multi do |multi| - multi.del(issue_ids_key) - multi.del(current_index_key) - multi.del(current_project_key) - multi.srem(CONCURRENT_RUNNING_REBALANCES_KEY, value) - multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.del(issue_ids_key) + multi.del(current_index_key) + multi.del(current_project_key) + multi.srem?(CONCURRENT_RUNNING_REBALANCES_KEY, value) + multi.set(self.class.recently_finished_key(rebalanced_container_type, rebalanced_container_id), true, ex: 1.hour) + end end end end @@ -136,9 +140,10 @@ module Gitlab current_rebalancing_containers.each do |string| container_type, container_id = string.split('/', 2).map(&:to_i) - if container_type == NAMESPACE + case container_type + when NAMESPACE namespace_ids << container_id - elsif container_type == PROJECT + when PROJECT project_ids << container_id end end diff --git a/lib/gitlab/json.rb b/lib/gitlab/json.rb index 823d6202b1e..8332e4f6d56 100644 --- a/lib/gitlab/json.rb +++ b/lib/gitlab/json.rb @@ -41,6 +41,11 @@ module Gitlab # as the underlying implementation of this varies wildly based on # the adapter in use. # + # This method does, in some situations, differ in the data it returns + # compared to .generate. Counter-intuitively, this is closest in + # terms of response to JSON.generate and to the default ActiveSupport + # .to_json method. + # # @param object [Object] the object to convert to JSON # @return [String] def dump(object) @@ -162,23 +167,11 @@ module Gitlab # @return [Boolean, String, Array, Hash, Object] # @raise [JSON::ParserError] def handle_legacy_mode!(data) - return data unless feature_table_exists? + return data unless Feature.feature_flags_available? return data unless Feature.enabled?(:json_wrapper_legacy_mode) raise parser_error if INVALID_LEGACY_TYPES.any? { |type| data.is_a?(type) } end - - # There are a variety of database errors possible when checking the feature - # flags at the wrong time during boot, e.g. during migrations. We don't care - # about these errors, we just need to ensure that we skip feature detection - # if they will fail. - # - # @return [Boolean] - def feature_table_exists? - Feature::FlipperFeature.table_exists? - rescue StandardError - false - end end # GrapeFormatter is a JSON formatter for the Grape API. @@ -263,5 +256,19 @@ module Gitlab buffer.string end end + + class RailsEncoder < ActiveSupport::JSON::Encoding::JSONGemEncoder + # Rails doesn't provide a way of changing the JSON adapter for + # render calls in controllers, so here we're overriding the parent + # class method to use our generator, and it's monkey-patched in + # config/initializers/active_support_json.rb + def stringify(jsonified) + Gitlab::Json.dump(jsonified) + rescue EncodingError => ex + # Raise the same error as the default implementation if we encounter + # an error. These are usually related to invalid UTF-8 errors. + raise JSON::GeneratorError, ex + end + end end end diff --git a/lib/gitlab/json_logger.rb b/lib/gitlab/json_logger.rb index d0dcd232ecc..7dbedef44ee 100644 --- a/lib/gitlab/json_logger.rb +++ b/lib/gitlab/json_logger.rb @@ -1,31 +1,52 @@ # frozen_string_literal: true +require 'labkit/logging' module Gitlab - class JsonLogger < ::Gitlab::Logger - def self.file_name_noext - raise NotImplementedError - end + class JsonLogger < ::Labkit::Logging::JsonLogger + class << self + def file_name_noext + raise NotImplementedError, "JsonLogger implementations must provide file_name_noext implementation" + end + + def file_name + file_name_noext + ".log" + end + + def debug(message) + build.debug(message) + end + + def error(message) + build.error(message) + end + + def warn(message) + build.warn(message) + end - def format_message(severity, timestamp, progname, message) - data = default_attributes - data[:severity] = severity - data[:time] = timestamp.utc.iso8601(3) - data[Labkit::Correlation::CorrelationId::LOG_KEY] = Labkit::Correlation::CorrelationId.current_id + def info(message) + build.info(message) + end - case message - when String - data[:message] = message - when Hash - data.merge!(message) + def build + Gitlab::SafeRequestStore[cache_key] ||= + new(full_log_path, level: log_level) end - Gitlab::Json.dump(data) + "\n" + def cache_key + "logger:" + full_log_path.to_s + end + + def full_log_path + Rails.root.join("log", file_name) + end end - protected + private - def default_attributes - {} + # Override Labkit's default impl, which uses the default Ruby platform json module. + def dump_json(data) + Gitlab::Json.dump(data) end end end diff --git a/lib/gitlab/kas.rb b/lib/gitlab/kas.rb index bf7b7f2d089..a1e290a54e6 100644 --- a/lib/gitlab/kas.rb +++ b/lib/gitlab/kas.rb @@ -34,7 +34,7 @@ module Gitlab end def version_info - Gitlab::VersionInfo.parse(version) + Gitlab::VersionInfo.parse(version, parse_suffix: true) end # Return GitLab KAS external_url diff --git a/lib/gitlab/kroki.rb b/lib/gitlab/kroki.rb index 6799be8e279..5fa77c1f1ba 100644 --- a/lib/gitlab/kroki.rb +++ b/lib/gitlab/kroki.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'asciidoctor/extensions/asciidoctor_kroki/version' require 'asciidoctor/extensions/asciidoctor_kroki/extension' module Gitlab diff --git a/lib/gitlab/manifest_import/metadata.rb b/lib/gitlab/manifest_import/metadata.rb index 6fe9bb10cdf..3747431c6a7 100644 --- a/lib/gitlab/manifest_import/metadata.rb +++ b/lib/gitlab/manifest_import/metadata.rb @@ -14,9 +14,11 @@ module Gitlab def save(repositories, group_id) Gitlab::Redis::SharedState.with do |redis| - redis.multi do |multi| - multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) - multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + redis.multi do |multi| + multi.set(key_for('repositories'), Gitlab::Json.dump(repositories), ex: EXPIRY_TIME) + multi.set(key_for('group_id'), group_id, ex: EXPIRY_TIME) + end end end end diff --git a/lib/gitlab/marginalia/comment.rb b/lib/gitlab/marginalia/comment.rb index aab58bfa211..1997ebb952b 100644 --- a/lib/gitlab/marginalia/comment.rb +++ b/lib/gitlab/marginalia/comment.rb @@ -45,6 +45,18 @@ module Gitlab def db_config_name ::Gitlab::Database.db_config_name(marginalia_adapter) end + + def console_hostname + return unless ::Gitlab::Runtime.console? + + @cached_console_hostname ||= Socket.gethostname # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + + def console_username + return unless ::Gitlab::Runtime.console? + + ENV['SUDO_USER'] || ENV['USER'] + end end end end diff --git a/lib/gitlab/markdown_cache/redis/store.rb b/lib/gitlab/markdown_cache/redis/store.rb index 752ab153f37..8cab069e1bf 100644 --- a/lib/gitlab/markdown_cache/redis/store.rb +++ b/lib/gitlab/markdown_cache/redis/store.rb @@ -10,9 +10,11 @@ module Gitlab results = {} Gitlab::Redis::Cache.with do |r| - r.pipelined do |pipeline| - subjects.each do |subject| - results[subject.cache_key] = new(subject).read(pipeline) + Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do + r.pipelined do |pipeline| + subjects.each do |subject| + results[subject.cache_key] = new(subject).read(pipeline) + end end end end @@ -28,7 +30,7 @@ module Gitlab def save(updates) @loaded = false - Gitlab::Redis::Cache.with do |r| + with_redis do |r| r.mapped_hmset(markdown_cache_key, updates) r.expire(markdown_cache_key, EXPIRES_IN) end @@ -40,7 +42,7 @@ module Gitlab if pipeline pipeline.mapped_hmget(markdown_cache_key, *fields) else - Gitlab::Redis::Cache.with do |r| + with_redis do |r| r.mapped_hmget(markdown_cache_key, *fields) end end @@ -64,6 +66,10 @@ module Gitlab "markdown_cache:#{@subject.cache_key}" end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/memory/watchdog.rb b/lib/gitlab/memory/watchdog.rb index 7007fdfe386..19dfc640b5d 100644 --- a/lib/gitlab/memory/watchdog.rb +++ b/lib/gitlab/memory/watchdog.rb @@ -54,6 +54,17 @@ module Gitlab init_prometheus_metrics end + ## + # Configuration for Watchdog, use like: + # + # watchdog.configure do |config| + # config.handler = Gitlab::Memory::Watchdog::TermProcessHandler + # config.sleep_time_seconds = 60 + # config.logger = Gitlab::AppLogger + # config.monitors do |stack| + # stack.push MyMonitorClass, args*, max_strikes:, kwargs**, &block + # end + # end def configure yield @configuration end @@ -125,7 +136,7 @@ module Gitlab end def process_rss_bytes - Gitlab::Metrics::System.memory_usage_rss + Gitlab::Metrics::System.memory_usage_rss[:total] end def worker_id diff --git a/lib/gitlab/memory/watchdog/configuration.rb b/lib/gitlab/memory/watchdog/configuration.rb index 2d84b083f55..793f75adf59 100644 --- a/lib/gitlab/memory/watchdog/configuration.rb +++ b/lib/gitlab/memory/watchdog/configuration.rb @@ -9,7 +9,7 @@ module Gitlab @monitors = [] end - def use(monitor_class, *args, **kwargs, &block) + def push(monitor_class, *args, **kwargs, &block) remove(monitor_class) @monitors.push(build_monitor_state(monitor_class, *args, **kwargs, &block)) end @@ -39,11 +39,12 @@ module Gitlab DEFAULT_SLEEP_TIME_SECONDS = 60 - attr_reader :monitors attr_writer :logger, :handler, :sleep_time_seconds - def initialize - @monitors = MonitorStack.new + def monitors + @monitor_stack ||= MonitorStack.new + yield @monitor_stack if block_given? + @monitor_stack end def handler diff --git a/lib/gitlab/memory/watchdog/configurator.rb b/lib/gitlab/memory/watchdog/configurator.rb new file mode 100644 index 00000000000..6d6f97dc8ba --- /dev/null +++ b/lib/gitlab/memory/watchdog/configurator.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + class Configurator + class << self + def configure_for_puma + lambda do |config| + sleep_time_seconds = ENV.fetch('GITLAB_MEMWD_SLEEP_TIME_SEC', 60).to_i + config.logger = Gitlab::AppLogger + config.handler = Gitlab::Memory::Watchdog::PumaHandler.new + config.sleep_time_seconds = sleep_time_seconds + config.monitors(&configure_monitors_for_puma) + end + end + + def configure_for_sidekiq + lambda do |config| + sleep_time_seconds = [ENV.fetch('SIDEKIQ_MEMORY_KILLER_CHECK_INTERVAL', 3).to_i, 2].max + config.logger = Sidekiq.logger + config.handler = Gitlab::Memory::Watchdog::TermProcessHandler.new + config.sleep_time_seconds = sleep_time_seconds + config.monitors(&configure_monitors_for_sidekiq) + end + end + + private + + def configure_monitors_for_puma + lambda do |stack| + max_strikes = ENV.fetch('GITLAB_MEMWD_MAX_STRIKES', 5).to_i + + if Gitlab::Utils.to_boolean(ENV['DISABLE_PUMA_WORKER_KILLER']) + max_heap_frag = ENV.fetch('GITLAB_MEMWD_MAX_HEAP_FRAG', 0.5).to_f + max_mem_growth = ENV.fetch('GITLAB_MEMWD_MAX_MEM_GROWTH', 3.0).to_f + + # stack.push MonitorClass, args*, max_strikes:, kwargs**, &block + stack.push Gitlab::Memory::Watchdog::Monitor::HeapFragmentation, + max_heap_fragmentation: max_heap_frag, + max_strikes: max_strikes + + stack.push Gitlab::Memory::Watchdog::Monitor::UniqueMemoryGrowth, + max_mem_growth: max_mem_growth, + max_strikes: max_strikes + else + memory_limit = ENV.fetch('PUMA_WORKER_MAX_MEMORY', 1200).to_i + + stack.push Gitlab::Memory::Watchdog::Monitor::RssMemoryLimit, + memory_limit: memory_limit, + max_strikes: max_strikes + end + end + end + + def configure_monitors_for_sidekiq + # NOP - At the moment we don't run watchdog for Sidekiq + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb index 7748c19c6d8..8f230980eac 100644 --- a/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb +++ b/lib/gitlab/memory/watchdog/monitor/heap_fragmentation.rb @@ -22,7 +22,7 @@ module Gitlab def call heap_fragmentation = Gitlab::Metrics::Memory.gc_heap_fragmentation - return { threshold_violated: false, payload: {} } unless heap_fragmentation > max_heap_fragmentation + return { threshold_violated: false, payload: {} } if heap_fragmentation <= max_heap_fragmentation { threshold_violated: true, payload: payload(heap_fragmentation) } end diff --git a/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb new file mode 100644 index 00000000000..3e7de024630 --- /dev/null +++ b/lib/gitlab/memory/watchdog/monitor/rss_memory_limit.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +module Gitlab + module Memory + class Watchdog + module Monitor + class RssMemoryLimit + attr_reader :memory_limit + + def initialize(memory_limit:) + @memory_limit = memory_limit + end + + def call + worker_rss = Gitlab::Metrics::System.memory_usage_rss[:total] + + return { threshold_violated: false, payload: {} } if worker_rss <= memory_limit + + { threshold_violated: true, payload: payload(worker_rss, memory_limit) } + end + + private + + def payload(worker_rss, memory_limit) + { + message: 'rss memory limit exceeded', + memwd_rss_bytes: worker_rss, + memwd_max_rss_bytes: memory_limit + } + end + end + end + end + end +end diff --git a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb index 2a1512c4cff..ce3477e6227 100644 --- a/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb +++ b/lib/gitlab/memory/watchdog/monitor/unique_memory_growth.rb @@ -16,7 +16,7 @@ module Gitlab reference_uss = reference_mem[:uss] memory_limit = max_mem_growth * reference_uss - return { threshold_violated: false, payload: {} } unless worker_uss > memory_limit + return { threshold_violated: false, payload: {} } if worker_uss <= memory_limit { threshold_violated: true, payload: payload(worker_uss, reference_uss, memory_limit) } end diff --git a/lib/gitlab/merge_requests/mergeability/check_result.rb b/lib/gitlab/merge_requests/mergeability/check_result.rb index a25156661af..fae4b721e1a 100644 --- a/lib/gitlab/merge_requests/mergeability/check_result.rb +++ b/lib/gitlab/merge_requests/mergeability/check_result.rb @@ -22,8 +22,8 @@ module Gitlab def self.from_hash(data) new( - status: data.fetch('status').to_sym, - payload: data.fetch('payload')) + status: data.fetch(:status).to_sym, + payload: data.fetch(:payload)) end def initialize(status:, payload: {}) diff --git a/lib/gitlab/merge_requests/mergeability/redis_interface.rb b/lib/gitlab/merge_requests/mergeability/redis_interface.rb index b0e739f91ff..1129fa639d8 100644 --- a/lib/gitlab/merge_requests/mergeability/redis_interface.rb +++ b/lib/gitlab/merge_requests/mergeability/redis_interface.rb @@ -7,16 +7,20 @@ module Gitlab VERSION = 1 def save_check(merge_check:, result_hash:) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.set(merge_check.cache_key + ":#{VERSION}", result_hash.to_json, ex: EXPIRATION) end end def retrieve_check(merge_check:) - Gitlab::Redis::Cache.with do |redis| - Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}")) + with_redis do |redis| + Gitlab::Json.parse(redis.get(merge_check.cache_key + ":#{VERSION}"), symbolize_keys: true) end end + + def with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord + end end end end diff --git a/lib/gitlab/metrics/dashboard/finder.rb b/lib/gitlab/metrics/dashboard/finder.rb index c8591a81a05..a4964ae0ebc 100644 --- a/lib/gitlab/metrics/dashboard/finder.rb +++ b/lib/gitlab/metrics/dashboard/finder.rb @@ -78,7 +78,7 @@ module Gitlab end def predefined_dashboard_services_for(project) - # Only list the self monitoring dashboard on the self monitoring project, + # Only list the self-monitoring dashboard on the self-monitoring project, # since it is the only dashboard (at time of writing) that shows data # about GitLab itself. if project.self_monitoring? diff --git a/lib/gitlab/metrics/global_search_slis.rb b/lib/gitlab/metrics/global_search_slis.rb index 3400a6c78ef..200c6eb4043 100644 --- a/lib/gitlab/metrics/global_search_slis.rb +++ b/lib/gitlab/metrics/global_search_slis.rb @@ -5,12 +5,12 @@ module Gitlab module GlobalSearchSlis class << self # The following targets are the 99.95th percentile of code searches - # gathered on 24-08-2022 + # gathered on 25-10-2022 # from https://log.gprd.gitlab.net/goto/0c89cd80-23af-11ed-8656-f5f2137823ba (internal only) - BASIC_CONTENT_TARGET_S = 7.031 - BASIC_CODE_TARGET_S = 21.903 - ADVANCED_CONTENT_TARGET_S = 4.865 - ADVANCED_CODE_TARGET_S = 13.546 + BASIC_CONTENT_TARGET_S = 8.812 + BASIC_CODE_TARGET_S = 27.538 + ADVANCED_CONTENT_TARGET_S = 2.452 + ADVANCED_CODE_TARGET_S = 15.52 def initialize_slis! Gitlab::Metrics::Sli::Apdex.initialize_sli(:global_search, possible_labels) diff --git a/lib/gitlab/metrics/loose_foreign_keys_slis.rb b/lib/gitlab/metrics/loose_foreign_keys_slis.rb new file mode 100644 index 00000000000..5d8245aa609 --- /dev/null +++ b/lib/gitlab/metrics/loose_foreign_keys_slis.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module Gitlab + module Metrics + module LooseForeignKeysSlis + class << self + def initialize_slis! + Gitlab::Metrics::Sli::Apdex.initialize_sli(:loose_foreign_key_clean_ups, possible_labels) + Gitlab::Metrics::Sli::ErrorRate.initialize_sli(:loose_foreign_key_clean_ups, possible_labels) + end + + def record_apdex(success:, db_config_name:) + Gitlab::Metrics::Sli::Apdex[:loose_foreign_key_clean_ups].increment( + labels: labels(db_config_name), + success: success + ) + end + + def record_error_rate(error:, db_config_name:) + Gitlab::Metrics::Sli::ErrorRate[:loose_foreign_key_clean_ups].increment( + labels: labels(db_config_name), + error: error + ) + end + + private + + def possible_labels + ::Gitlab::Database.db_config_names.map do |db_config_name| + { + db_config_name: db_config_name, + feature_category: :database + } + end + end + + def labels(db_config_name) + { + db_config_name: db_config_name, + feature_category: :database + } + end + end + end + end +end diff --git a/lib/gitlab/metrics/method_call.rb b/lib/gitlab/metrics/method_call.rb index c6b0a0c5e76..f39ec9cc8ab 100644 --- a/lib/gitlab/metrics/method_call.rb +++ b/lib/gitlab/metrics/method_call.rb @@ -39,7 +39,6 @@ module Gitlab docstring 'Method calls real duration' label_keys label_keys buckets [0.01, 0.05, 0.1, 0.5, 1] - with_feature :prometheus_metrics_method_instrumentation end end diff --git a/lib/gitlab/metrics/samplers/base_sampler.rb b/lib/gitlab/metrics/samplers/base_sampler.rb index b2a9de21145..e62a62a935e 100644 --- a/lib/gitlab/metrics/samplers/base_sampler.rb +++ b/lib/gitlab/metrics/samplers/base_sampler.rb @@ -46,11 +46,11 @@ module Gitlab # 2. Don't sample data at the same interval two times in a row. def sleep_interval while step = @interval_steps.sample - if step != @last_step - @last_step = step + next if step == @last_step - return @interval + @last_step - end + @last_step = step + + return @interval + @last_step end end diff --git a/lib/gitlab/metrics/samplers/ruby_sampler.rb b/lib/gitlab/metrics/samplers/ruby_sampler.rb index 4fe338ffc7f..5a7ca6b6c04 100644 --- a/lib/gitlab/metrics/samplers/ruby_sampler.rb +++ b/lib/gitlab/metrics/samplers/ruby_sampler.rb @@ -35,6 +35,8 @@ module Gitlab process_cpu_seconds_total: ::Gitlab::Metrics.gauge(metric_name(:process, :cpu_seconds_total), 'Process CPU seconds total'), process_max_fds: ::Gitlab::Metrics.gauge(metric_name(:process, :max_fds), 'Process max fds'), process_resident_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_memory_bytes), 'Memory used (RSS)', labels), + process_resident_anon_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_anon_memory_bytes), 'Anonymous memory used (RSS)', labels), + process_resident_file_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :resident_file_memory_bytes), 'File backed memory used (RSS)', labels), process_unique_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :unique_memory_bytes), 'Memory used (USS)', labels), process_proportional_memory_bytes: ::Gitlab::Metrics.gauge(metric_name(:process, :proportional_memory_bytes), 'Memory used (PSS)', labels), process_start_time_seconds: ::Gitlab::Metrics.gauge(metric_name(:process, :start_time_seconds), 'Process start time seconds'), @@ -95,7 +97,10 @@ module Gitlab end def set_memory_usage_metrics - metrics[:process_resident_memory_bytes].set(labels, System.memory_usage_rss) + rss = System.memory_usage_rss + metrics[:process_resident_memory_bytes].set(labels, rss[:total]) + metrics[:process_resident_anon_memory_bytes].set(labels, rss[:anon]) + metrics[:process_resident_file_memory_bytes].set(labels, rss[:file]) if Gitlab::Utils.to_boolean(ENV['enable_memory_uss_pss'] || '1') memory_uss_pss = System.memory_usage_uss_pss diff --git a/lib/gitlab/metrics/system.rb b/lib/gitlab/metrics/system.rb index affadc4274c..9b0ae84dec2 100644 --- a/lib/gitlab/metrics/system.rb +++ b/lib/gitlab/metrics/system.rb @@ -18,7 +18,9 @@ module Gitlab PRIVATE_PAGES_PATTERN = /^(Private_Clean|Private_Dirty|Private_Hugetlb):\s+(?<value>\d+)/.freeze PSS_PATTERN = /^Pss:\s+(?<value>\d+)/.freeze - RSS_PATTERN = /VmRSS:\s+(?<value>\d+)/.freeze + RSS_TOTAL_PATTERN = /^VmRSS:\s+(?<value>\d+)/.freeze + RSS_ANON_PATTERN = /^RssAnon:\s+(?<value>\d+)/.freeze + RSS_FILE_PATTERN = /^RssFile:\s+(?<value>\d+)/.freeze MAX_OPEN_FILES_PATTERN = /Max open files\s*(?<value>\d+)/.freeze MEM_TOTAL_PATTERN = /^MemTotal:\s+(?<value>\d+) (.+)/.freeze @@ -27,7 +29,7 @@ module Gitlab { version: RUBY_DESCRIPTION, gc_stat: GC.stat, - memory_rss: memory_usage_rss, + memory_rss: memory_usage_rss[:total], memory_uss: proportional_mem[:uss], memory_pss: proportional_mem[:pss], time_cputime: cpu_time, @@ -38,7 +40,21 @@ module Gitlab # Returns the given process' RSS (resident set size) in bytes. def memory_usage_rss(pid: 'self') - sum_matches(PROC_STATUS_PATH % pid, rss: RSS_PATTERN)[:rss].kilobytes + results = { total: 0, anon: 0, file: 0 } + + safe_yield_procfile(PROC_STATUS_PATH % pid) do |io| + io.each_line do |line| + if (value = parse_metric_value(line, RSS_TOTAL_PATTERN)) > 0 + results[:total] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_ANON_PATTERN)) > 0 + results[:anon] = value.kilobytes + elsif (value = parse_metric_value(line, RSS_FILE_PATTERN)) > 0 + results[:file] = value.kilobytes + end + end + end + + results end # Returns the given process' USS/PSS (unique/proportional set size) in bytes. @@ -115,9 +131,7 @@ module Gitlab safe_yield_procfile(proc_file) do |io| io.each_line do |line| patterns.each do |metric, pattern| - match = line.match(pattern) - value = match&.named_captures&.fetch('value', 0) - results[metric] += value.to_i + results[metric] += parse_metric_value(line, pattern) end end end @@ -125,6 +139,13 @@ module Gitlab results end + def parse_metric_value(line, pattern) + match = line.match(pattern) + return 0 unless match + + match.named_captures.fetch('value', 0).to_i + end + def proc_stat_entries safe_yield_procfile(PROC_STAT_PATH) do |io| io.read.split(' ') diff --git a/lib/gitlab/nav/top_nav_view_model_builder.rb b/lib/gitlab/nav/top_nav_view_model_builder.rb index a8e25708107..8cb2729ff61 100644 --- a/lib/gitlab/nav/top_nav_view_model_builder.rb +++ b/lib/gitlab/nav/top_nav_view_model_builder.rb @@ -42,13 +42,10 @@ module Gitlab def build menu = @menu_builder.build - hide_menu_text = Feature.enabled?(:new_navbar_layout) - menu.merge({ views: @views, shortcuts: @shortcuts, - menuTitle: (_('Menu') unless hide_menu_text), - menuTooltip: (_('Main menu') if hide_menu_text) + menuTooltip: _('Main menu') }.compact) end end diff --git a/lib/gitlab/observability.rb b/lib/gitlab/observability.rb new file mode 100644 index 00000000000..8dde60a73be --- /dev/null +++ b/lib/gitlab/observability.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Observability + module_function + + def observability_url + return ENV['OVERRIDE_OBSERVABILITY_URL'] if ENV['OVERRIDE_OBSERVABILITY_URL'] + # TODO Make observability URL configurable https://gitlab.com/gitlab-org/opstrace/opstrace-ui/-/issues/80 + return 'https://observe.staging.gitlab.com' if Gitlab.staging? + + 'https://observe.gitlab.com' + end + end +end diff --git a/lib/gitlab/octokit/middleware.rb b/lib/gitlab/octokit/middleware.rb index a3c0fdcf467..a92860f7eb8 100644 --- a/lib/gitlab/octokit/middleware.rb +++ b/lib/gitlab/octokit/middleware.rb @@ -8,7 +8,11 @@ module Gitlab end def call(env) - Gitlab::UrlBlocker.validate!(env[:url], allow_localhost: allow_local_requests?, allow_local_network: allow_local_requests?) + Gitlab::UrlBlocker.validate!(env[:url], + schemes: %w[http https], + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests? + ) @app.call(env) end diff --git a/lib/gitlab/pagination/gitaly_keyset_pager.rb b/lib/gitlab/pagination/gitaly_keyset_pager.rb index d4de2791195..6235874132f 100644 --- a/lib/gitlab/pagination/gitaly_keyset_pager.rb +++ b/lib/gitlab/pagination/gitaly_keyset_pager.rb @@ -35,11 +35,12 @@ module Gitlab def keyset_pagination_enabled?(finder) return false unless params[:pagination] == "keyset" - if finder.is_a?(BranchesFinder) + case finder + when BranchesFinder Feature.enabled?(:branch_list_keyset_pagination, project) - elsif finder.is_a?(TagsFinder) + when TagsFinder true - elsif finder.is_a?(::Repositories::TreeFinder) + when ::Repositories::TreeFinder Feature.enabled?(:repository_tree_gitaly_pagination, project) else false @@ -49,11 +50,12 @@ module Gitlab def paginate_first_page?(finder) return false unless params[:page].blank? || params[:page].to_i == 1 - if finder.is_a?(BranchesFinder) + case finder + when BranchesFinder Feature.enabled?(:branch_list_keyset_pagination, project) - elsif finder.is_a?(TagsFinder) + when TagsFinder true - elsif finder.is_a?(::Repositories::TreeFinder) + when ::Repositories::TreeFinder Feature.enabled?(:repository_tree_gitaly_pagination, project) else false diff --git a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb index 51f38c1da58..4f79a3593f4 100644 --- a/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb +++ b/lib/gitlab/pagination/keyset/in_operator_optimization/strategies/record_loader_strategy.rb @@ -39,15 +39,15 @@ module Gitlab def verify_order_by_attributes_on_model!(model, order_by_columns) order_by_columns.map(&:column).each do |column| - unless model.columns_hash[column.attribute_name.to_s] - text = <<~TEXT + next if model.columns_hash[column.attribute_name.to_s] + + text = <<~TEXT The "RecordLoaderStrategy" does not support the following ORDER BY column because it's not available on the \"#{model.table_name}\" table: #{column.attribute_name} Omit the "finder_query" parameter to use the "OrderValuesLoaderStrategy". - TEXT - raise text - end + TEXT + raise text end end end diff --git a/lib/gitlab/pagination_delegate.rb b/lib/gitlab/pagination_delegate.rb new file mode 100644 index 00000000000..05aaff5bbfc --- /dev/null +++ b/lib/gitlab/pagination_delegate.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +module Gitlab + class PaginationDelegate # rubocop:disable Gitlab/NamespacedClass + DEFAULT_PER_PAGE = Kaminari.config.default_per_page + MAX_PER_PAGE = Kaminari.config.max_per_page + + def initialize(page:, per_page:, count:, options: {}) + @count = count + @options = { default_per_page: DEFAULT_PER_PAGE, + max_per_page: MAX_PER_PAGE }.merge(options) + + @per_page = sanitize_per_page(per_page) + @page = sanitize_page(page) + end + + def total_count + @count + end + + def total_pages + (total_count.to_f / @per_page).ceil + end + + def next_page + current_page + 1 unless last_page? + end + + def prev_page + current_page - 1 unless first_page? + end + + def current_page + @page + end + + def limit_value + @per_page + end + + def first_page? + current_page == 1 + end + + def last_page? + current_page >= total_pages + end + + def offset + (current_page - 1) * limit_value + end + + private + + def sanitize_per_page(per_page) + return @options[:default_per_page] unless per_page && per_page > 0 + + [@options[:max_per_page], per_page].min + end + + def sanitize_page(page) + return 1 unless page && page > 1 + + [total_pages, page].min + end + end +end diff --git a/lib/gitlab/patch/sidekiq_cron_poller.rb b/lib/gitlab/patch/sidekiq_cron_poller.rb new file mode 100644 index 00000000000..c9eae2f899f --- /dev/null +++ b/lib/gitlab/patch/sidekiq_cron_poller.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +# Patch to address https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/1932 +# It restores the behavior of `poll_internal_average` to the one from Sidekiq 6.5.7 +# when the cron poll interval is not configured. +# (see https://github.com/mperham/sidekiq/blob/v6.5.7/lib/sidekiq/scheduled.rb#L176-L178) +require 'sidekiq/version' +require 'sidekiq/cron/version' + +if Gem::Version.new(Sidekiq::VERSION) != Gem::Version.new('6.5.7') + raise 'New version of sidekiq detected, please remove or update this patch' +end + +if Gem::Version.new(Sidekiq::Cron::VERSION) != Gem::Version.new('1.8.0') + raise 'New version of sidekiq-cron detected, please remove or update this patch' +end + +module Gitlab + module Patch + module SidekiqCronPoller + def poll_interval_average(count) + Gitlab.config.cron_jobs.poll_interval || @config[:poll_interval_average] || scaled_poll_interval(count) # rubocop:disable Gitlab/ModuleWithInstanceVariables + end + end + end +end diff --git a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb index fbc77113875..79c00a48336 100644 --- a/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb +++ b/lib/gitlab/performance_bar/redis_adapter_when_peek_enabled.rb @@ -19,7 +19,7 @@ module Gitlab def enqueue_stats_job(request_id) return unless Feature.enabled?(:performance_bar_stats, type: :ops) - @client.sadd(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) + @client.sadd?(GitlabPerformanceBarStatsWorker::STATS_KEY, request_id) return unless uuid = Gitlab::ExclusiveLease.new( GitlabPerformanceBarStatsWorker::LEASE_KEY, diff --git a/lib/gitlab/project_search_results.rb b/lib/gitlab/project_search_results.rb index fb9447f9665..8cc96970ebd 100644 --- a/lib/gitlab/project_search_results.rb +++ b/lib/gitlab/project_search_results.rb @@ -117,7 +117,7 @@ module Gitlab end def blobs(limit: count_limit) - return [] unless Ability.allowed?(@current_user, :download_code, @project) + return [] unless Ability.allowed?(@current_user, :read_code, @project) @blobs ||= Gitlab::FileFinder.new(project, repository_project_ref).find(query, content_match_cutoff: limit) end @@ -153,7 +153,7 @@ module Gitlab end def find_commits(query, limit:) - return [] unless Ability.allowed?(@current_user, :download_code, @project) + return [] unless Ability.allowed?(@current_user, :read_code, @project) commits = find_commits_by_message(query, limit: limit) commit_by_sha = find_commit_by_sha(query) diff --git a/lib/gitlab/project_template.rb b/lib/gitlab/project_template.rb index 6673940ccf3..51a5bedc44b 100644 --- a/lib/gitlab/project_template.rb +++ b/lib/gitlab/project_template.rb @@ -28,6 +28,14 @@ module Gitlab "#{preview}.git" end + def project_host + return unless preview + + uri = URI.parse(preview) + uri.path = "" + uri.to_s + end + def project_path URI.parse(preview).path.delete_prefix('/') end diff --git a/lib/gitlab/qa.rb b/lib/gitlab/qa.rb new file mode 100644 index 00000000000..c47a8982901 --- /dev/null +++ b/lib/gitlab/qa.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module Gitlab + module Qa + def self.user_agent + ENV['GITLAB_QA_USER_AGENT'] + end + + def self.request?(request_user_agent) + return false unless Gitlab.com? + return false unless request_user_agent.present? + return false unless user_agent.present? + + ActiveSupport::SecurityUtils.secure_compare(request_user_agent, user_agent) + end + end +end diff --git a/lib/gitlab/query_limiting/transaction.rb b/lib/gitlab/query_limiting/transaction.rb index 46c0a0ddf7a..498da38e268 100644 --- a/lib/gitlab/query_limiting/transaction.rb +++ b/lib/gitlab/query_limiting/transaction.rb @@ -68,14 +68,14 @@ module Gitlab GEO_NODES_LOAD = 'SELECT 1 AS one FROM "geo_nodes" LIMIT 1' LICENSES_LOAD = 'SELECT "licenses".* FROM "licenses" ORDER BY "licenses"."id"' - ATTR_INTROSPECTION = %r/SELECT .*\ba.attname\b.* (FROM|JOIN) pg_attribute a/m.freeze + SCHEMA_INTROSPECTION = %r/SELECT.*(FROM|JOIN) (pg_attribute|pg_class)/m.freeze # queries can be safely ignored if they are amoritized in regular usage # (i.e. only requested occasionally and otherwise cached). def ignorable?(sql) return true if sql&.include?(GEO_NODES_LOAD) return true if sql&.include?(LICENSES_LOAD) - return true if ATTR_INTROSPECTION =~ sql + return true if SCHEMA_INTROSPECTION.match?(sql) false end diff --git a/lib/gitlab/quick_actions/issuable_actions.rb b/lib/gitlab/quick_actions/issuable_actions.rb index 3b85d6952a1..0b37c80dc5f 100644 --- a/lib/gitlab/quick_actions/issuable_actions.rb +++ b/lib/gitlab/quick_actions/issuable_actions.rb @@ -12,16 +12,13 @@ module Gitlab included do # Issue, MergeRequest, Epic: quick actions definitions desc do - _('Close this %{quick_action_target}') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Close this %{quick_action_target}') % { quick_action_target: target_issuable_name } end explanation do - _('Closes this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Closes this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Closed this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Closed this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -35,15 +32,15 @@ module Gitlab desc do _('Reopen this %{quick_action_target}') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end explanation do _('Reopens this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end execution_message do _('Reopened this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -170,12 +167,10 @@ module Gitlab desc { _('Subscribe') } explanation do - _('Subscribes to this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Subscribes to this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Subscribed to this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Subscribed to this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -188,12 +183,10 @@ module Gitlab desc { _('Unsubscribe') } explanation do - _('Unsubscribes from this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Unsubscribes from this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end execution_message do - _('Unsubscribed from this %{quick_action_target}.') % - { quick_action_target: quick_action_target.to_ability_name.humanize(capitalize: false) } + _('Unsubscribed from this %{quick_action_target}.') % { quick_action_target: target_issuable_name } end types ::Issuable condition do @@ -266,6 +259,16 @@ module Gitlab end end + desc { _("Make %{type} confidential") % { type: target_issuable_name } } + explanation { _("Makes this %{type} confidential.") % { type: target_issuable_name } } + types ::Issuable + condition { quick_action_target.supports_confidentiality? && can_make_confidential? } + command :confidential do + @updates[:confidential] = true + + @execution_message[:confidential] = confidential_execution_message + end + private def find_severity(severity_param) @@ -315,6 +318,29 @@ module Gitlab _('Removed all labels.') end end + + def target_issuable_name + quick_action_target.to_ability_name.humanize(capitalize: false) + end + + def can_make_confidential? + confidentiality_not_supported = quick_action_target.respond_to?(:issue_type_supports?) && + !quick_action_target.issue_type_supports?(:confidentiality) + + return false if confidentiality_not_supported + + !quick_action_target.confidential? && current_user.can?(:set_confidentiality, quick_action_target) + end + + def confidential_execution_message + confidential_error_message.presence || _("Made this %{type} confidential.") % { type: target_issuable_name } + end + + def confidential_error_message + return unless quick_action_target.respond_to?(:confidentiality_errors) + + quick_action_target.confidentiality_errors.join("\n") + end end end end diff --git a/lib/gitlab/quick_actions/issue_actions.rb b/lib/gitlab/quick_actions/issue_actions.rb index 4883c649a62..e74c58e45b1 100644 --- a/lib/gitlab/quick_actions/issue_actions.rb +++ b/lib/gitlab/quick_actions/issue_actions.rb @@ -161,23 +161,6 @@ module Gitlab @execution_message[:move] = message end - desc { _('Make issue confidential') } - explanation do - _('Makes this issue confidential.') - end - execution_message do - _('Made this issue confidential.') - end - types Issue - condition do - quick_action_target.issue_type_supports?(:confidentiality) && - !quick_action_target.confidential? && - current_user.can?(:set_confidentiality, quick_action_target) - end - command :confidential do - @updates[:confidential] = true - end - desc { _('Create a merge request') } explanation do |branch_name = nil| if branch_name diff --git a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb index a0faf8dd460..8b1ff5d298a 100644 --- a/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb +++ b/lib/gitlab/quick_actions/issue_and_merge_request_actions.rb @@ -161,7 +161,7 @@ module Gitlab parse_params do |raw_duration| Gitlab::TimeTrackingFormatter.parse(raw_duration) end - command :estimate do |time_estimate| + command :estimate, :estimate_time do |time_estimate| if time_estimate @updates[:time_estimate] = time_estimate end @@ -184,7 +184,7 @@ module Gitlab parse_params do |raw_time_date| Gitlab::QuickActions::SpendTimeAndDateSeparator.new(raw_time_date).execute end - command :spend, :spent do |time_spent, time_spent_date| + command :spend, :spent, :spend_time do |time_spent, time_spent_date| if time_spent @updates[:spend_time] = { duration: time_spent, @@ -202,7 +202,7 @@ module Gitlab quick_action_target.persisted? && current_user.can?(:"admin_#{quick_action_target.to_ability_name}", project) end - command :remove_estimate do + command :remove_estimate, :remove_time_estimate do @updates[:time_estimate] = 0 end diff --git a/lib/gitlab/redis/multi_store.rb b/lib/gitlab/redis/multi_store.rb index a7c36786d2d..12cb1fc6153 100644 --- a/lib/gitlab/redis/multi_store.rb +++ b/lib/gitlab/redis/multi_store.rb @@ -52,6 +52,7 @@ module Gitlab del flushdb rpush + eval ).freeze PIPELINED_COMMANDS = %i( diff --git a/lib/gitlab/reference_extractor.rb b/lib/gitlab/reference_extractor.rb index f914123a94d..c5798bec0d7 100644 --- a/lib/gitlab/reference_extractor.rb +++ b/lib/gitlab/reference_extractor.rb @@ -42,10 +42,10 @@ module Gitlab @references[type] ||= references(type) end - if %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) - define_method("#{type}_ids") do - @references[type] ||= references(type, ids_only: true) - end + next unless %w(mentioned_user mentioned_group mentioned_project).include?(type.to_s) + + define_method("#{type}_ids") do + @references[type] ||= references(type, ids_only: true) end end diff --git a/lib/gitlab/request_forgery_protection.rb b/lib/gitlab/request_forgery_protection.rb index 258c904290d..d5e80053772 100644 --- a/lib/gitlab/request_forgery_protection.rb +++ b/lib/gitlab/request_forgery_protection.rb @@ -10,6 +10,14 @@ module Gitlab class Controller < ActionController::Base protect_from_forgery with: :exception, prepend: true + def initialize + super + + # Squelch noisy and unnecessary "Can't verify CSRF token authenticity." messages. + # X-Csrf-Token is only one authentication mechanism for API helpers. + self.logger = ActiveSupport::Logger.new(File::NULL) + end + def index head :ok end diff --git a/lib/gitlab/runtime.rb b/lib/gitlab/runtime.rb index 6d95cb9a87b..7e9fb82fb8b 100644 --- a/lib/gitlab/runtime.rb +++ b/lib/gitlab/runtime.rb @@ -42,7 +42,7 @@ module Gitlab end def sidekiq? - !!(defined?(::Sidekiq) && Sidekiq.server?) + !!(defined?(::Sidekiq) && Sidekiq.try(:server?)) end def rake? @@ -94,7 +94,7 @@ module Gitlab # # These threads execute Sidekiq client middleware when jobs # are enqueued and those can access DB / Redis. - threads += Sidekiq.options[:concurrency] + 2 + threads += Sidekiq[:concurrency] + 2 end if puma? diff --git a/lib/gitlab/search/recent_items.rb b/lib/gitlab/search/recent_items.rb index 593148025e1..7a16b5dfc87 100644 --- a/lib/gitlab/search/recent_items.rb +++ b/lib/gitlab/search/recent_items.rb @@ -33,7 +33,7 @@ module Gitlab end def search(term) - finder.new(user, search: term, in: 'title') + finder.new(user, search: term, in: 'title', skip_full_text_search_project_condition: true) .execute .limit(SEARCH_LIMIT).reorder(nil).id_in_ordered(latest_ids) # rubocop: disable CodeReuse/ActiveRecord end diff --git a/lib/gitlab/service_desk_email.rb b/lib/gitlab/service_desk_email.rb index 14f07140825..bc49efafdda 100644 --- a/lib/gitlab/service_desk_email.rb +++ b/lib/gitlab/service_desk_email.rb @@ -3,8 +3,10 @@ module Gitlab module ServiceDeskEmail class << self - def enabled? - !!config&.enabled && config&.address.present? + include Gitlab::Email::Common + + def config + Gitlab.config.service_desk_email end def key_from_address(address) @@ -14,20 +16,10 @@ module Gitlab Gitlab::IncomingEmail.key_from_address(address, wildcard_address: wildcard_address) end - def config - Gitlab.config.service_desk_email - end - def address_for_key(key) return if config.address.blank? - config.address.sub(Gitlab::IncomingEmail::WILDCARD_PLACEHOLDER, key) - end - - def key_from_fallback_message_id(mail_id) - message_id_regexp = /\Areply\-(.+)@#{Gitlab.config.gitlab.host}\z/ - - mail_id[message_id_regexp, 1] + config.address.sub(WILDCARD_PLACEHOLDER, key) end end end diff --git a/lib/gitlab/set_cache.rb b/lib/gitlab/set_cache.rb index c7818cb3418..3d2ff5a68d2 100644 --- a/lib/gitlab/set_cache.rb +++ b/lib/gitlab/set_cache.rb @@ -34,7 +34,7 @@ module Gitlab def write(key, value) with do |redis| redis.pipelined do |pipeline| - pipeline.sadd(cache_key(key), value) + pipeline.sadd?(cache_key(key), value) pipeline.expire(cache_key(key), expires_in) end diff --git a/lib/gitlab/shard_health_cache.rb b/lib/gitlab/shard_health_cache.rb index eeb0cc75ef9..a34e4e9c8d1 100644 --- a/lib/gitlab/shard_health_cache.rb +++ b/lib/gitlab/shard_health_cache.rb @@ -7,17 +7,17 @@ module Gitlab # Clears the Redis set storing the list of healthy shards def self.clear - Gitlab::Redis::Cache.with { |redis| redis.del(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.del(HEALTHY_SHARDS_KEY) } end # Updates the list of healthy shards using a Redis set # # shards - An array of shard names to store def self.update(shards) - Gitlab::Redis::Cache.with do |redis| + with_redis do |redis| redis.multi do |m| m.del(HEALTHY_SHARDS_KEY) - shards.each { |shard_name| m.sadd(HEALTHY_SHARDS_KEY, shard_name) } + m.sadd(HEALTHY_SHARDS_KEY, shards) unless shards.blank? m.expire(HEALTHY_SHARDS_KEY, HEALTHY_SHARDS_TIMEOUT) end end @@ -25,19 +25,23 @@ module Gitlab # Returns an array of strings of healthy shards def self.cached_healthy_shards - Gitlab::Redis::Cache.with { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.smembers(HEALTHY_SHARDS_KEY) } end # Checks whether the given shard name is in the list of healthy shards. # # shard_name - The string to check def self.healthy_shard?(shard_name) - Gitlab::Redis::Cache.with { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } + with_redis { |redis| redis.sismember(HEALTHY_SHARDS_KEY, shard_name) } end # Returns the number of healthy shards in the Redis set def self.healthy_shard_count - Gitlab::Redis::Cache.with { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + with_redis { |redis| redis.scard(HEALTHY_SHARDS_KEY) } + end + + def self.with_redis(&block) + Gitlab::Redis::Cache.with(&block) # rubocop:disable CodeReuse/ActiveRecord end end end diff --git a/lib/gitlab/shell.rb b/lib/gitlab/shell.rb index b167afe589a..bc59d4ce943 100644 --- a/lib/gitlab/shell.rb +++ b/lib/gitlab/shell.rb @@ -14,6 +14,11 @@ module Gitlab class Shell Error = Class.new(StandardError) + PERMITTED_ACTIONS = %w[ + mv_repository remove_repository add_namespace rm_namespace mv_namespace + repository_exists? + ].freeze + class << self # Retrieve GitLab Shell secret token # diff --git a/lib/gitlab/sidekiq_config.rb b/lib/gitlab/sidekiq_config.rb index 3e7bdfbe89a..7e2a934b3dd 100644 --- a/lib/gitlab/sidekiq_config.rb +++ b/lib/gitlab/sidekiq_config.rb @@ -162,7 +162,7 @@ module Gitlab # the current Sidekiq process def current_worker_queue_mappings worker_queue_mappings - .select { |worker, queue| Sidekiq.options[:queues].include?(queue) } + .select { |worker, queue| Sidekiq[:queues].include?(queue) } .to_h end diff --git a/lib/gitlab/sidekiq_daemon/memory_killer.rb b/lib/gitlab/sidekiq_daemon/memory_killer.rb index b8f86b92844..d5227e7a007 100644 --- a/lib/gitlab/sidekiq_daemon/memory_killer.rb +++ b/lib/gitlab/sidekiq_daemon/memory_killer.rb @@ -118,9 +118,9 @@ module Gitlab return unless enabled? # Tell sidekiq to restart itself - # Keep extra safe to wait `Sidekiq.options[:timeout] + 2` seconds before SIGKILL + # Keep extra safe to wait `Sidekiq[:timeout] + 2` seconds before SIGKILL refresh_state(:shutting_down) - signal_and_wait(Sidekiq.options[:timeout] + 2, 'SIGTERM', 'gracefully shut down') + signal_and_wait(Sidekiq[:timeout] + 2, 'SIGTERM', 'gracefully shut down') return unless enabled? # Ideally we should never reach this condition @@ -221,7 +221,7 @@ module Gitlab end def get_rss_kb - Gitlab::Metrics::System.memory_usage_rss / 1.kilobytes + Gitlab::Metrics::System.memory_usage_rss[:total] / 1.kilobytes end def get_soft_limit_rss_kb diff --git a/lib/gitlab/sidekiq_middleware/arguments_logger.rb b/lib/gitlab/sidekiq_middleware/arguments_logger.rb index fe5213fc5d7..2c506786d83 100644 --- a/lib/gitlab/sidekiq_middleware/arguments_logger.rb +++ b/lib/gitlab/sidekiq_middleware/arguments_logger.rb @@ -3,8 +3,10 @@ module Gitlab module SidekiqMiddleware class ArgumentsLogger + include Sidekiq::ServerMiddleware + def call(worker, job, queue) - Sidekiq.logger.info "arguments: #{Gitlab::Json.dump(job['args'])}" + logger.info "arguments: #{Gitlab::Json.dump(job['args'])}" yield end end diff --git a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb index d42bd672bac..357e9d41187 100644 --- a/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb +++ b/lib/gitlab/sidekiq_middleware/duplicate_jobs/duplicate_job.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require 'digest' +require 'msgpack' module Gitlab module SidekiqMiddleware @@ -20,23 +21,8 @@ module Gitlab include Gitlab::Utils::StrongMemoize DEFAULT_DUPLICATE_KEY_TTL = 6.hours - WAL_LOCATION_TTL = 60.seconds - MAX_REDIS_RETRIES = 5 DEFAULT_STRATEGY = :until_executing STRATEGY_NONE = :none - DEDUPLICATED_FLAG_VALUE = 1 - - LUA_SET_WAL_SCRIPT = <<~EOS - local key, wal, offset, ttl = KEYS[1], ARGV[1], tonumber(ARGV[2]), ARGV[3] - local existing_offset = redis.call("LINDEX", key, -1) - if existing_offset == false then - redis.call("RPUSH", key, wal, offset) - redis.call("EXPIRE", key, ttl) - elseif offset > tonumber(existing_offset) then - redis.call("LSET", key, 0, wal) - redis.call("LSET", key, -1, offset) - end - EOS attr_reader :existing_jid @@ -60,66 +46,76 @@ module Gitlab # This method will return the jid that was set in redis def check!(expiry = duplicate_key_ttl) - read_jid = nil - read_wal_locations = {} - - with_redis do |redis| - redis.multi do |multi| - multi.set(idempotency_key, jid, ex: expiry, nx: true) - read_wal_locations = check_existing_wal_locations!(multi, expiry) - read_jid = multi.get(idempotency_key) - end + my_cookie = { + 'jid' => jid, + 'offsets' => {}, + 'wal_locations' => {}, + 'existing_wal_locations' => job_wal_locations + } + + # There are 3 possible scenarios. In order of decreasing likelyhood: + # 1. SET NX succeeds. + # 2. SET NX fails, GET succeeds. + # 3. SET NX fails, the key expires and GET fails. In this case we must retry. + actual_cookie = {} + while actual_cookie.empty? + set_succeeded = with_redis { |r| r.set(cookie_key, my_cookie.to_msgpack, nx: true, ex: expiry) } + actual_cookie = set_succeeded ? my_cookie : get_cookie end job['idempotency_key'] = idempotency_key - # We need to fetch values since the read_wal_locations and read_jid were obtained inside transaction, under redis.multi command. - self.existing_wal_locations = read_wal_locations.transform_values(&:value) - self.existing_jid = read_jid.value + self.existing_wal_locations = actual_cookie['existing_wal_locations'] + self.existing_jid = actual_cookie['jid'] end def update_latest_wal_location! return unless job_wal_locations.present? - with_redis do |redis| - redis.multi do |multi| - job_wal_locations.each do |connection_name, location| - multi.eval( - LUA_SET_WAL_SCRIPT, - keys: [wal_location_key(connection_name)], - argv: [location, pg_wal_lsn_diff(connection_name).to_i, WAL_LOCATION_TTL] - ) - end - end + argv = [] + job_wal_locations.each do |connection_name, location| + argv += [connection_name, pg_wal_lsn_diff(connection_name), location] end + + with_redis { |r| r.eval(UPDATE_WAL_COOKIE_SCRIPT, keys: [cookie_key], argv: argv) } end + # Generally speaking, updating a Redis key by deserializing and + # serializing it on the Redis server is bad for performance. However in + # the case of DuplicateJobs we know that key updates are rare, and the + # most common operations are setting, getting and deleting the key. The + # aim of this design is to make the common operations as fast as + # possible. + UPDATE_WAL_COOKIE_SCRIPT = <<~LUA + local cookie_msgpack = redis.call("get", KEYS[1]) + if not cookie_msgpack then + return + end + local cookie = cmsgpack.unpack(cookie_msgpack) + + for i = 1, #ARGV, 3 do + local connection = ARGV[i] + local current_offset = cookie.offsets[connection] + local new_offset = tonumber(ARGV[i+1]) + if not current_offset or current_offset < new_offset then + cookie.offsets[connection] = new_offset + cookie.wal_locations[connection] = ARGV[i+2] + end + end + + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + LUA + def latest_wal_locations return {} unless job_wal_locations.present? strong_memoize(:latest_wal_locations) do - read_wal_locations = {} - - with_redis do |redis| - redis.multi do |multi| - job_wal_locations.keys.each do |connection_name| - read_wal_locations[connection_name] = multi.lindex(wal_location_key(connection_name), 0) - end - end - end - read_wal_locations.transform_values(&:value).compact + get_cookie.fetch('wal_locations', {}) end end def delete! - Gitlab::Instrumentation::RedisClusterValidator.allow_cross_slot_commands do - with_redis do |redis| - redis.multi do |multi| - multi.del(idempotency_key, deduplicated_flag_key) - delete_wal_locations!(multi) - end - end - end + with_redis { |redis| redis.del(cookie_key) } end def reschedule @@ -141,17 +137,21 @@ module Gitlab def set_deduplicated_flag!(expiry = duplicate_key_ttl) return unless reschedulable? - with_redis do |redis| - redis.set(deduplicated_flag_key, DEDUPLICATED_FLAG_VALUE, ex: expiry, nx: true) - end + with_redis { |redis| redis.eval(DEDUPLICATED_SCRIPT, keys: [cookie_key]) } end - def should_reschedule? - return false unless reschedulable? - - with_redis do |redis| - redis.get(deduplicated_flag_key).present? + DEDUPLICATED_SCRIPT = <<~LUA + local cookie_msgpack = redis.call("get", KEYS[1]) + if not cookie_msgpack then + return end + local cookie = cmsgpack.unpack(cookie_msgpack) + cookie.deduplicated = "1" + redis.call("set", KEYS[1], cmsgpack.pack(cookie), "ex", redis.call("ttl", KEYS[1])) + LUA + + def should_reschedule? + reschedulable? && get_cookie['deduplicated'].present? end def scheduled_at @@ -186,31 +186,12 @@ module Gitlab @worker_klass ||= worker_class_name.to_s.safe_constantize end - def delete_wal_locations!(redis) - job_wal_locations.keys.each do |connection_name| - redis.del(wal_location_key(connection_name)) - redis.del(existing_wal_location_key(connection_name)) - end - end - - def check_existing_wal_locations!(redis, expiry) - read_wal_locations = {} - - job_wal_locations.each do |connection_name, location| - key = existing_wal_location_key(connection_name) - redis.set(key, location, ex: expiry, nx: true) - read_wal_locations[connection_name] = redis.get(key) - end - - read_wal_locations - end - def job_wal_locations job['wal_locations'] || {} end def pg_wal_lsn_diff(connection_name) - model = Gitlab::Database.database_base_models[connection_name] + model = Gitlab::Database.database_base_models[connection_name.to_sym] model.connection.load_balancer.wal_diff( job_wal_locations[connection_name], @@ -238,22 +219,18 @@ module Gitlab job['jid'] end - def existing_wal_location_key(connection_name) - "#{idempotency_key}:#{connection_name}:existing_wal_location" + def cookie_key + "#{idempotency_key}:cookie:v2" end - def wal_location_key(connection_name) - "#{idempotency_key}:#{connection_name}:wal_location" + def get_cookie + with_redis { |redis| MessagePack.unpack(redis.get(cookie_key) || "\x80") } end def idempotency_key @idempotency_key ||= job['idempotency_key'] || "#{namespace}:#{idempotency_hash}" end - def deduplicated_flag_key - "#{idempotency_key}:deduplicate_flag" - end - def idempotency_hash Digest::SHA256.hexdigest(idempotency_string) end diff --git a/lib/gitlab/sidekiq_middleware/retry_error.rb b/lib/gitlab/sidekiq_middleware/retry_error.rb new file mode 100644 index 00000000000..372213a8e6a --- /dev/null +++ b/lib/gitlab/sidekiq_middleware/retry_error.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module SidekiqMiddleware + # Sidekiq retry error that won't be reported to Sentry + # Use it when a job retry is an expected behavior + RetryError = Class.new(StandardError) + end +end diff --git a/lib/gitlab/sidekiq_middleware/server_metrics.rb b/lib/gitlab/sidekiq_middleware/server_metrics.rb index 3dd5355d3a3..e36f61be3b3 100644 --- a/lib/gitlab/sidekiq_middleware/server_metrics.rb +++ b/lib/gitlab/sidekiq_middleware/server_metrics.rb @@ -43,7 +43,7 @@ module Gitlab def initialize_process_metrics metrics = self.metrics - metrics[:sidekiq_concurrency].set({}, Sidekiq.options[:concurrency].to_i) + metrics[:sidekiq_concurrency].set({}, Sidekiq[:concurrency].to_i) return unless ::Feature.enabled?(:sidekiq_job_completion_metric_initialize) diff --git a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb index bce295d8ba5..f7e0553e536 100644 --- a/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb +++ b/lib/gitlab/sidekiq_middleware/size_limiter/compressor.rb @@ -33,7 +33,7 @@ module Gitlab validate_args!(job) job.except!(ORIGINAL_SIZE_KEY, COMPRESSED_KEY) - job['args'] = Sidekiq.load_json(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) + job['args'] = Gitlab::Json.load(Zlib::Inflate.inflate(Base64.strict_decode64(job['args'].first))) rescue Zlib::Error raise PayloadDecompressionError, 'Fail to decompress Sidekiq job payload' end diff --git a/lib/gitlab/sidekiq_migrate_jobs.rb b/lib/gitlab/sidekiq_migrate_jobs.rb index 62d62bf82c4..2467dd7ca43 100644 --- a/lib/gitlab/sidekiq_migrate_jobs.rb +++ b/lib/gitlab/sidekiq_migrate_jobs.rb @@ -3,16 +3,18 @@ module Gitlab class SidekiqMigrateJobs LOG_FREQUENCY = 1_000 + LOG_FREQUENCY_QUEUES = 10 - attr_reader :sidekiq_set, :logger + attr_reader :logger, :mappings - def initialize(sidekiq_set, logger: nil) - @sidekiq_set = sidekiq_set + # mappings is a hash of WorkerClassName => target_queue_name + def initialize(mappings, logger: nil) + @mappings = mappings @logger = logger end - # mappings is a hash of WorkerClassName => target_queue_name - def execute(mappings) + # Migrate jobs in SortedSets, i.e. scheduled and retry sets. + def migrate_set(sidekiq_set) source_queues_regex = Regexp.union(mappings.keys) cursor = 0 scanned = 0 @@ -33,7 +35,7 @@ module Gitlab next unless job.match?(source_queues_regex) - job_hash = Sidekiq.load_json(job) + job_hash = Gitlab::Json.load(job) destination_queue = mappings[job_hash['class']] next unless mappings.has_key?(job_hash['class']) @@ -41,7 +43,7 @@ module Gitlab job_hash['queue'] = destination_queue - migrated += migrate_job(job, score, job_hash) + migrated += migrate_job_in_set(sidekiq_set, job, score, job_hash) end end while cursor.to_i != 0 @@ -53,14 +55,54 @@ module Gitlab } end + # Migrates jobs from queues that are outside the mappings + def migrate_queues + routing_rules_queues = mappings.values.uniq + logger&.info("List of queues based on routing rules: #{routing_rules_queues}") + Sidekiq.redis do |conn| + # Redis 6 supports conn.scan_each(match: "queue:*", type: 'list') + conn.scan_each(match: "queue:*") do |key| + # Redis 5 compatibility + next unless conn.type(key) == 'list' + + queue_from = key.split(':', 2).last + next if routing_rules_queues.include?(queue_from) + + logger&.info("Migrating #{queue_from} queue") + + migrated = 0 + while queue_length(queue_from) > 0 + begin + if migrated >= 0 && migrated % LOG_FREQUENCY_QUEUES == 0 + logger&.info("Migrating from #{queue_from}. Total: #{queue_length(queue_from)}. Migrated: #{migrated}.") + end + + job = conn.rpop "queue:#{queue_from}" + job_hash = Gitlab::Json.load(job) + next unless mappings.has_key?(job_hash['class']) + + destination_queue = mappings[job_hash['class']] + job_hash['queue'] = destination_queue + conn.lpush("queue:#{destination_queue}", Gitlab::Json.dump(job_hash)) + migrated += 1 + rescue JSON::ParserError + logger&.error("Unmarshal JSON payload from SidekiqMigrateJobs failed. Job: #{job}") + next + end + end + logger&.info("Finished migrating #{queue_from} queue") + end + end + end + private - def migrate_job(job, score, job_hash) + def migrate_job_in_set(sidekiq_set, job, score, job_hash) Sidekiq.redis do |connection| removed = connection.zrem(sidekiq_set, job) if removed - connection.zadd(sidekiq_set, score, Sidekiq.dump_json(job_hash)) + connection.zadd(sidekiq_set, score, Gitlab::Json.dump(job_hash)) 1 else @@ -68,5 +110,11 @@ module Gitlab end end end + + def queue_length(queue_name) + Sidekiq.redis do |conn| + conn.llen("queue:#{queue_name}") + end + end end end diff --git a/lib/gitlab/slash_commands/application_help.rb b/lib/gitlab/slash_commands/application_help.rb index 1a92346be15..bfdb65a816d 100644 --- a/lib/gitlab/slash_commands/application_help.rb +++ b/lib/gitlab/slash_commands/application_help.rb @@ -3,14 +3,9 @@ module Gitlab module SlashCommands class ApplicationHelp < BaseCommand - def initialize(project, params) - @project = project - @params = params - end - def execute Gitlab::SlashCommands::Presenters::Help - .new(project, commands) + .new(project, commands, params) .present(trigger, params[:text]) end @@ -21,7 +16,11 @@ module Gitlab end def commands - Gitlab::SlashCommands::Command.commands + Gitlab::SlashCommands::Command.new( + project, + chat_name, + params + ).commands end end end diff --git a/lib/gitlab/slash_commands/command.rb b/lib/gitlab/slash_commands/command.rb index 239479f99d2..265eda46489 100644 --- a/lib/gitlab/slash_commands/command.rb +++ b/lib/gitlab/slash_commands/command.rb @@ -3,8 +3,8 @@ module Gitlab module SlashCommands class Command < BaseCommand - def self.commands - [ + def commands + commands = [ Gitlab::SlashCommands::IssueShow, Gitlab::SlashCommands::IssueNew, Gitlab::SlashCommands::IssueSearch, @@ -14,6 +14,12 @@ module Gitlab Gitlab::SlashCommands::Deploy, Gitlab::SlashCommands::Run ] + + if Feature.enabled?(:incident_declare_slash_command, current_user) + commands << Gitlab::SlashCommands::IncidentManagement::IncidentNew + end + + commands end def execute @@ -44,7 +50,7 @@ module Gitlab private def available_commands - self.class.commands.keep_if do |klass| + commands.keep_if do |klass| klass.available?(project) end end diff --git a/lib/gitlab/slash_commands/incident_management/incident_command.rb b/lib/gitlab/slash_commands/incident_management/incident_command.rb new file mode 100644 index 00000000000..3fa08621777 --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_command.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentCommand < BaseCommand + def self.available?(project) + true + end + + def collection + IssuesFinder.new(current_user, project_id: project.id, issue_types: :incident).execute + end + end + end + end +end + +Gitlab::SlashCommands::IncidentManagement::IncidentCommand.prepend_mod diff --git a/lib/gitlab/slash_commands/incident_management/incident_new.rb b/lib/gitlab/slash_commands/incident_management/incident_new.rb new file mode 100644 index 00000000000..722fcff151d --- /dev/null +++ b/lib/gitlab/slash_commands/incident_management/incident_new.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module IncidentManagement + class IncidentNew < IncidentCommand + def self.help_message + 'incident declare' + end + + def self.allowed?(project, user) + Feature.enabled?(:incident_declare_slash_command, user) && can?(user, :create_incident, project) + end + + def self.match(text) + text == 'incident declare' + end + + private + + def presenter + Gitlab::SlashCommands::Presenters::IncidentManagement::IncidentNew.new + end + end + end + end +end + +Gitlab::SlashCommands::IncidentManagement::IncidentNew.prepend_mod diff --git a/lib/gitlab/slash_commands/presenters/help.rb b/lib/gitlab/slash_commands/presenters/help.rb index 71bc0dc0123..61b36308d20 100644 --- a/lib/gitlab/slash_commands/presenters/help.rb +++ b/lib/gitlab/slash_commands/presenters/help.rb @@ -4,9 +4,10 @@ module Gitlab module SlashCommands module Presenters class Help < Presenters::Base - def initialize(project, commands) + def initialize(project, commands, params = {}) @project = project @commands = commands + @params = params end def present(trigger, text) @@ -66,7 +67,13 @@ module Gitlab def full_commands_message(trigger) list = @commands - .map { |command| "#{trigger} #{command.help_message}" } + .map do |command| + if command < Gitlab::SlashCommands::IncidentManagement::IncidentCommand + "#{@params[:command]} #{command.help_message}" + else + "#{trigger} #{command.help_message}" + end + end .join("\n") <<~MESSAGE diff --git a/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb new file mode 100644 index 00000000000..5030c8282db --- /dev/null +++ b/lib/gitlab/slash_commands/presenters/incident_management/incident_new.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module SlashCommands + module Presenters + module IncidentManagement + class IncidentNew < Presenters::Base + def present(message) + ephemeral_response(text: message) + end + end + end + end + end +end diff --git a/lib/gitlab/sql/pattern.rb b/lib/gitlab/sql/pattern.rb index ca7ae429986..d13ccde8576 100644 --- a/lib/gitlab/sql/pattern.rb +++ b/lib/gitlab/sql/pattern.rb @@ -6,7 +6,7 @@ module Gitlab extend ActiveSupport::Concern MIN_CHARS_FOR_PARTIAL_MATCHING = 3 - REGEX_QUOTED_WORD = /(?<=\A| )"[^"]+"(?= |\z)/.freeze + REGEX_QUOTED_TERM = /(?<=\A| )"[^"]+"(?= |\z)/.freeze class_methods do def fuzzy_search(query, columns, use_minimum_char_limit: true) @@ -40,12 +40,14 @@ module Gitlab # lower_exact_match - When set to `true` we'll fall back to using # `LOWER(column) = query` instead of using `ILIKE`. def fuzzy_arel_match(column, query, lower_exact_match: false, use_minimum_char_limit: true) + return unless query.is_a?(String) + query = query.squish return unless query.present? arel_column = column.is_a?(Arel::Attributes::Attribute) ? column : arel_table[column] - words = select_fuzzy_words(query, use_minimum_char_limit: use_minimum_char_limit) + words = select_fuzzy_terms(query, use_minimum_char_limit: use_minimum_char_limit) if words.any? words.map { |word| arel_column.matches(to_pattern(word, use_minimum_char_limit: use_minimum_char_limit)) }.reduce(:and) @@ -62,19 +64,21 @@ module Gitlab end end - def select_fuzzy_words(query, use_minimum_char_limit: true) - quoted_words = query.scan(REGEX_QUOTED_WORD) - - query = quoted_words.reduce(query) { |q, quoted_word| q.sub(quoted_word, '') } - - words = query.split - - quoted_words.map! { |quoted_word| quoted_word[1..-2] } + def select_fuzzy_terms(query, use_minimum_char_limit: true) + terms = Gitlab::SQL::Pattern.split_query_to_search_terms(query) + terms.select { |term| partial_matching?(term, use_minimum_char_limit: use_minimum_char_limit) } + end + end - words.concat(quoted_words) + def self.split_query_to_search_terms(query) + quoted_terms = [] - words.select { |word| partial_matching?(word, use_minimum_char_limit: use_minimum_char_limit) } + query = query.gsub(REGEX_QUOTED_TERM) do |quoted_term| + quoted_terms << quoted_term + "" end + + query.split + quoted_terms.map { |quoted_term| quoted_term[1..-2] } end end end diff --git a/lib/gitlab/template/base_template.rb b/lib/gitlab/template/base_template.rb index 31e11f73fe7..ededc3db18e 100644 --- a/lib/gitlab/template/base_template.rb +++ b/lib/gitlab/template/base_template.rb @@ -47,6 +47,8 @@ module Gitlab { key: key, name: name, content: content } end + alias_method :as_json, :to_json + def <=>(other) name <=> other.name end diff --git a/lib/gitlab/tracking/helpers/weak_password_error_event.rb b/lib/gitlab/tracking/helpers/weak_password_error_event.rb new file mode 100644 index 00000000000..beb6119e3f7 --- /dev/null +++ b/lib/gitlab/tracking/helpers/weak_password_error_event.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +module Gitlab + module Tracking + module Helpers + module WeakPasswordErrorEvent + # Tracks information if a user record has a weak password. + # No-op unless the error is present. + # + # Captures a minimal set of information, so that GitLab + # remains unaware of which users / demographics are attempting + # to choose weak passwords. + def track_weak_password_error(user, controller, method_name) + return unless user.errors[:password].grep(/must not contain commonly used combinations.*/).any? + + Gitlab::Tracking.event( + 'Gitlab::Tracking::Helpers::WeakPasswordErrorEvent', + 'track_weak_password_error', + controller: controller, + method: method_name + ) + end + end + end + end +end diff --git a/lib/gitlab/url_builder.rb b/lib/gitlab/url_builder.rb index a6d6cffec17..e203fb486e7 100644 --- a/lib/gitlab/url_builder.rb +++ b/lib/gitlab/url_builder.rb @@ -52,6 +52,8 @@ module Gitlab wiki_page_url(object.wiki, object, **options) when ::DesignManagement::Design design_url(object, **options) + when ::Packages::Package + package_url(object, **options) else raise NotImplementedError, "No URL builder defined for #{object.inspect}" end @@ -133,6 +135,17 @@ module Gitlab instance.project_design_management_designs_raw_image_url(design.project, design, ref, **options) end end + + def package_url(package, **options) + project = package.project + + if package.infrastructure_package? + return instance.project_infrastructure_registry_url(project, package, +**options) + end + + instance.project_package_url(project, package, **options) + end end end end diff --git a/lib/gitlab/usage/metric_definition.rb b/lib/gitlab/usage/metric_definition.rb index d6b1e62c84f..065ede75c60 100644 --- a/lib/gitlab/usage/metric_definition.rb +++ b/lib/gitlab/usage/metric_definition.rb @@ -4,9 +4,9 @@ module Gitlab module Usage class MetricDefinition METRIC_SCHEMA_PATH = Rails.root.join('config', 'metrics', 'schema.json') - SKIP_VALIDATION_STATUSES = %w[deprecated removed].to_set.freeze - AVAILABLE_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze - VALID_SERVICE_PING_STATUSES = %w[active data_available implemented deprecated broken].to_set.freeze + SKIP_VALIDATION_STATUS = 'removed' + AVAILABLE_STATUSES = %w[active broken].to_set.freeze + VALID_SERVICE_PING_STATUSES = %w[active broken].to_set.freeze InvalidError = Class.new(RuntimeError) @@ -144,7 +144,7 @@ module Gitlab end def skip_validation? - !!attributes[:skip_validation] || @skip_validation || SKIP_VALIDATION_STATUSES.include?(attributes[:status]) + !!attributes[:skip_validation] || @skip_validation || attributes[:status] == SKIP_VALIDATION_STATUS end end end diff --git a/lib/gitlab/usage/metrics/aggregates.rb b/lib/gitlab/usage/metrics/aggregates.rb index a32c413dba8..02d9fa74289 100644 --- a/lib/gitlab/usage/metrics/aggregates.rb +++ b/lib/gitlab/usage/metrics/aggregates.rb @@ -7,14 +7,13 @@ module Gitlab UNION_OF_AGGREGATED_METRICS = 'OR' INTERSECTION_OF_AGGREGATED_METRICS = 'AND' ALLOWED_METRICS_AGGREGATIONS = [UNION_OF_AGGREGATED_METRICS, INTERSECTION_OF_AGGREGATED_METRICS].freeze - AGGREGATED_METRICS_PATH = Rails.root.join('config/metrics/aggregates/*.yml') AggregatedMetricError = Class.new(StandardError) UnknownAggregationOperator = Class.new(AggregatedMetricError) UnknownAggregationSource = Class.new(AggregatedMetricError) DisallowedAggregationTimeFrame = Class.new(AggregatedMetricError) DATABASE_SOURCE = 'database' - REDIS_SOURCE = 'redis' + REDIS_SOURCE = 'redis_hll' SOURCES = { DATABASE_SOURCE => Sources::PostgresHll, diff --git a/lib/gitlab/usage/metrics/aggregates/aggregate.rb b/lib/gitlab/usage/metrics/aggregates/aggregate.rb index cd72f16d46d..78f1ddc8a29 100644 --- a/lib/gitlab/usage/metrics/aggregates/aggregate.rb +++ b/lib/gitlab/usage/metrics/aggregates/aggregate.rb @@ -8,22 +8,9 @@ module Gitlab include Gitlab::Usage::TimeFrame def initialize(recorded_at) - @aggregated_metrics = load_metrics(AGGREGATED_METRICS_PATH) @recorded_at = recorded_at end - def all_time_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::ALL_TIME_TIME_FRAME_NAME) - end - - def monthly_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME) - end - - def weekly_data - aggregated_metrics_data(Gitlab::Usage::TimeFrame::SEVEN_DAYS_TIME_FRAME_NAME) - end - def calculate_count_for_aggregation(aggregation:, time_frame:) with_validate_configuration(aggregation, time_frame) do source = SOURCES[aggregation[:source]] @@ -40,16 +27,7 @@ module Gitlab private - attr_accessor :aggregated_metrics, :recorded_at - - def aggregated_metrics_data(time_frame) - aggregated_metrics.each_with_object({}) do |aggregation, data| - next if aggregation[:feature_flag] && Feature.disabled?(aggregation[:feature_flag], type: :development) - next unless aggregation[:time_frame].include?(time_frame) - - data[aggregation[:name]] = calculate_count_for_aggregation(aggregation: aggregation, time_frame: time_frame) - end - end + attr_accessor :recorded_at def with_validate_configuration(aggregation, time_frame) source = aggregation[:source] @@ -83,16 +61,6 @@ module Gitlab Gitlab::Utils::UsageData::FALLBACK end - def load_metrics(wildcard) - Dir[wildcard].each_with_object([]) do |path, metrics| - metrics.push(*load_yaml_from_path(path)) - end - end - - def load_yaml_from_path(path) - YAML.safe_load(File.read(path), aliases: true)&.map(&:with_indifferent_access) - end - def time_constraints(time_frame) case time_frame when Gitlab::Usage::TimeFrame::TWENTY_EIGHT_DAYS_TIME_FRAME_NAME @@ -108,5 +76,3 @@ module Gitlab end end end - -Gitlab::Usage::Metrics::Aggregates::Aggregate.prepend_mod_with('Gitlab::Usage::Metrics::Aggregates::Aggregate') diff --git a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb index 63ead5a8cb0..66be7a7b64e 100644 --- a/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/aggregated_metric.rb @@ -25,7 +25,7 @@ module Gitlab def initialize(metric_definition) super - @source = parse_data_source_to_legacy_value(metric_definition) + @source = metric_definition[:data_source] @aggregate = options.fetch(:aggregate, {}) end @@ -48,15 +48,6 @@ module Gitlab attr_accessor :source, :aggregate - # TODO: This method is a temporary measure that - # handles backwards compatibility until - # point 5 from is resolved https://gitlab.com/gitlab-org/gitlab/-/issues/370963#implementation - def parse_data_source_to_legacy_value(metric_definition) - return 'redis' if metric_definition[:data_source] == 'redis_hll' - - metric_definition[:data_source] - end - def aggregate_config { source: source, diff --git a/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb new file mode 100644 index 00000000000..a7f8bca8e08 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/count_merge_request_authors_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class CountMergeRequestAuthorsMetric < DatabaseMetric + operation :distinct_count, column: :author_id + + relation { MergeRequest } + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb index 6dec0349a38..f0d5298870c 100644 --- a/lib/gitlab/usage/metrics/instrumentations/database_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/database_metric.rb @@ -34,10 +34,10 @@ module Gitlab @metric_finish = block end - def relation(&block) - return @metric_relation&.call unless block + def relation(relation_proc = nil, &block) + return unless relation_proc || block - @metric_relation = block + @metric_relation = (relation_proc || block) end def metric_options(&block) @@ -106,7 +106,11 @@ module Gitlab end def relation - self.class.metric_relation.call.where(time_constraints) + if self.class.metric_relation.arity == 1 + self.class.metric_relation.call(options) + else + self.class.metric_relation.call + end.where(time_constraints) end def time_constraints diff --git a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_metric.rb index 0c421dc3311..c7cf6c57059 100644 --- a/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_disabled_metric.rb +++ b/lib/gitlab/usage/metrics/instrumentations/distinct_count_projects_with_expiration_policy_metric.rb @@ -4,7 +4,7 @@ module Gitlab module Usage module Metrics module Instrumentations - class DistinctCountProjectsWithExpirationPolicyDisabledMetric < DatabaseMetric + class DistinctCountProjectsWithExpirationPolicyMetric < DatabaseMetric operation :distinct_count, column: :project_id start { Project.minimum(:id) } @@ -12,7 +12,11 @@ module Gitlab cache_start_and_finish_as :project_id - relation { ::ContainerExpirationPolicy.where(enabled: false) } + relation ->(options) do + options.each_with_object(::ContainerExpirationPolicy.all) do |(key, value), ar_relation| + ar_relation.where!(key => value) + end + end end end end diff --git a/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb b/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb new file mode 100644 index 00000000000..c05664aa9c8 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/dormant_user_period_setting_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DormantUserPeriodSettingMetric < GenericMetric + value do + ::Gitlab::CurrentSettings.deactivate_dormant_users_period + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb b/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb new file mode 100644 index 00000000000..82d8570276a --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/dormant_user_setting_enabled_metric.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class DormantUserSettingEnabledMetric < GenericMetric + value do + ::Gitlab::CurrentSettings.deactivate_dormant_users + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb new file mode 100644 index 00000000000..b1a2de29fd7 --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_cta_clicked_metric.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InProductMarketingEmailCtaClickedMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + unless track.in?(allowed_track) + raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" + end + + return if series.in?(allowed_series) + + raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" + end + + relation { Users::InProductMarketingEmail } + + private + + def relation + scope = super.where.not(cta_clicked_at: nil) + scope = scope.where(series: series) + scope.where(track: track) + end + + def track + options[:track] + end + + def series + options[:series] + end + + def allowed_track + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys + end + + def allowed_series + @allowed_series ||= begin + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) + 0.upto(series_amount - 1).to_a + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb new file mode 100644 index 00000000000..50dec606d9b --- /dev/null +++ b/lib/gitlab/usage/metrics/instrumentations/in_product_marketing_email_sent_metric.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module Instrumentations + class InProductMarketingEmailSentMetric < DatabaseMetric + operation :count + + def initialize(metric_definition) + super + + unless track.in?(allowed_track) + raise ArgumentError, "track '#{track}' must be one of: #{allowed_track.join(', ')}" + end + + return if series.in?(allowed_series) + + raise ArgumentError, "series '#{series}' must be one of: #{allowed_series.join(', ')}" + end + + relation { Users::InProductMarketingEmail } + + private + + def relation + scope = super + scope = scope.where(series: series) + scope.where(track: track) + end + + def track + options[:track] + end + + def series + options[:series] + end + + def allowed_track + Users::InProductMarketingEmail::ACTIVE_TRACKS.keys + end + + def allowed_series + @allowed_series ||= begin + series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) + 0.upto(series_amount - 1).to_a + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/name_suggestion.rb b/lib/gitlab/usage/metrics/name_suggestion.rb index 238a7a51a20..44723b6f3d4 100644 --- a/lib/gitlab/usage/metrics/name_suggestion.rb +++ b/lib/gitlab/usage/metrics/name_suggestion.rb @@ -7,6 +7,7 @@ module Gitlab FREE_TEXT_METRIC_NAME = "<please fill metric name>" REDIS_EVENT_METRIC_NAME = "<please fill metric name, suggested format is: {subject}_{verb}{ing|ed}_{object} eg: users_creating_epics or merge_requests_viewed_in_single_file_mode>" CONSTRAINTS_PROMPT_TEMPLATE = "<adjective describing: '%{constraints}'>" + EMPTY_CONSTRAINT = "()" class << self def for(operation, relation: nil, column: nil) @@ -52,7 +53,8 @@ module Gitlab end arel = arel_query(relation: relation, column: arel_column, distinct: distinct) - constraints = parse_constraints(relation: relation, arel: arel) + where_constraints = parse_where_constraints(relation: relation, arel: arel) + having_constraints = parse_having_constraints(relation: relation, arel: arel) # In some cases due to performance reasons metrics are instrumented with joined relations # where relation listed in FROM statement is not the one that includes counted attribute @@ -66,23 +68,35 @@ module Gitlab # count_environment_id_from_clusters_with_deployments actual_source = parse_source(relation, arel_column) - append_constraints_prompt(actual_source, [constraints], parts) + append_constraints_prompt(actual_source, [where_constraints], [having_constraints], parts) parts << actual_source - parts += process_joined_relations(actual_source, arel, relation, constraints) + parts += process_joined_relations(actual_source, arel, relation, where_constraints) parts.compact.join('_').delete('"') end - def append_constraints_prompt(target, constraints, parts) - applicable_constraints = constraints.select { |constraint| constraint.include?(target) } + def append_constraints_prompt(target, where_constraints, having_constraints, parts) + where_constraints.select! do |constraint| + constraint.include?(target) + end + having_constraints.delete(EMPTY_CONSTRAINT) + applicable_constraints = where_constraints + having_constraints return unless applicable_constraints.any? parts << CONSTRAINTS_PROMPT_TEMPLATE % { constraints: applicable_constraints.join(' AND ') } end - def parse_constraints(relation:, arel:) + def parse_where_constraints(relation:, arel:) + connection = relation.connection + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::WhereConstraints + .new(connection) + .accept(arel, collector(connection)) + .value + end + + def parse_having_constraints(relation:, arel:) connection = relation.connection - ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::Constraints + ::Gitlab::Usage::Metrics::NamesSuggestions::RelationParsers::HavingConstraints .new(connection) .accept(arel, collector(connection)) .value @@ -152,7 +166,7 @@ module Gitlab subtree.each do |parent, children| parts << "<#{conjunction}>" join_constraints = joins.find { |join| join[:source] == parent }&.dig(:constraints) - append_constraints_prompt(parent, [wheres, join_constraints].compact, parts) + append_constraints_prompt(parent, [wheres, join_constraints].compact, [], parts) parts << parent collect_join_parts(relations: children, joins: joins, wheres: wheres, parts: parts, conjunctions: conjunctions) end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb new file mode 100644 index 00000000000..8dd3b1ff5c6 --- /dev/null +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/having_constraints.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Gitlab + module Usage + module Metrics + module NamesSuggestions + module RelationParsers + class HavingConstraints < ::Arel::Visitors::PostgreSQL + # rubocop:disable Naming/MethodName + def visit_Arel_Nodes_SelectCore(object, collector) + collect_nodes_for(object.havings, collector, "") || collector + end + # rubocop:enable Naming/MethodName + + def quote(value) + value.to_s + end + + def quote_table_name(name) + name.to_s + end + + def quote_column_name(name) + name.to_s + end + end + end + end + end + end +end diff --git a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb index 199395e4b20..9f829067214 100644 --- a/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/constraints.rb +++ b/lib/gitlab/usage/metrics/names_suggestions/relation_parsers/where_constraints.rb @@ -5,7 +5,7 @@ module Gitlab module Metrics module NamesSuggestions module RelationParsers - class Constraints < ::Arel::Visitors::PostgreSQL + class WhereConstraints < ::Arel::Visitors::PostgreSQL # rubocop:disable Naming/MethodName def visit_Arel_Nodes_SelectCore(object, collector) collect_nodes_for(object.wheres, collector, "") || collector @@ -13,15 +13,15 @@ module Gitlab # rubocop:enable Naming/MethodName def quote(value) - "#{value}" + value.to_s end def quote_table_name(name) - "#{name}" + name.to_s end def quote_column_name(name) - "#{name}" + name.to_s end end end diff --git a/lib/gitlab/usage_data.rb b/lib/gitlab/usage_data.rb index 87ccb9a31da..5021dac453f 100644 --- a/lib/gitlab/usage_data.rb +++ b/lib/gitlab/usage_data.rb @@ -30,7 +30,6 @@ module Gitlab deployment_minimum_id deployment_maximum_id auth_providers - aggregated_metrics recorded_at ).freeze @@ -157,11 +156,9 @@ module Gitlab }.merge( runners_usage, integrations_usage, - usage_counters, user_preferences_usage, container_expiration_policies_usage, - service_desk_counts, - email_campaign_counts + service_desk_counts ).tap do |data| data[:snippets] = add(data[:personal_snippets], data[:project_snippets]) end @@ -261,16 +258,6 @@ module Gitlab } end - # @return [Hash<Symbol, Integer>] - def usage_counters - usage_data_counters.map { |counter| redis_usage_data(counter) }.reduce({}, :merge) - end - - # @return [Array<#totals>] An array of objects that respond to `#totals` - def usage_data_counters - Gitlab::UsageDataCounters.unmigrated_counters - end - def components_usage_data { git: { version: alt_usage_data(fallback: { major: -1 }) { Gitlab::Git.version } }, @@ -349,17 +336,13 @@ module Gitlab # rubocop: disable UsageData/LargeTable base = ::ContainerExpirationPolicy.active # rubocop: enable UsageData/LargeTable - results[:projects_with_expiration_policy_enabled] = distinct_count(base, :project_id, start: start, finish: finish) # rubocop: disable UsageData/LargeTable - %i[keep_n cadence older_than].each do |option| - ::ContainerExpirationPolicy.public_send("#{option}_options").keys.each do |value| # rubocop: disable GitlabSecurity/PublicSend - results["projects_with_expiration_policy_enabled_with_#{option}_set_to_#{value}".to_sym] = distinct_count(base.where(option => value), :project_id, start: start, finish: finish) - end + ::ContainerExpirationPolicy.older_than_options.keys.each do |value| + results["projects_with_expiration_policy_enabled_with_older_than_set_to_#{value}".to_sym] = distinct_count(base.where(older_than: value), :project_id, start: start, finish: finish) end # rubocop: enable UsageData/LargeTable - results[:projects_with_expiration_policy_enabled_with_keep_n_unset] = distinct_count(base.where(keep_n: nil), :project_id, start: start, finish: finish) results[:projects_with_expiration_policy_enabled_with_older_than_unset] = distinct_count(base.where(older_than: nil), :project_id, start: start, finish: finish) results @@ -632,21 +615,16 @@ module Gitlab { redis_hll_counters: ::Gitlab::UsageDataCounters::HLLRedisCounter.unique_events_data } end - def aggregated_metrics_data - { - counts_weekly: { aggregated_metrics: aggregated_metrics.weekly_data }, - counts_monthly: { aggregated_metrics: aggregated_metrics.monthly_data }, - counts: aggregated_metrics - .all_time_data - .to_h { |key, value| ["aggregate_#{key}".to_sym, value.round] } - } - end - def action_monthly_active_users(time_period) + counter = Gitlab::UsageDataCounters::EditorUniqueCounter date_range = { date_from: time_period[:created_at].first, date_to: time_period[:created_at].last } - event_monthly_active_users(date_range) - .merge!(ide_monthly_active_users(date_range)) + { + action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, + action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, + action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, + action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } + } end def with_duration @@ -688,7 +666,6 @@ module Gitlab .merge(usage_activity_by_stage) .merge(usage_activity_by_stage(:usage_activity_by_stage_monthly, monthly_time_range_db_params)) .merge(redis_hll_counters) - .deep_merge(aggregated_metrics_data) end def metric_time_period(time_period) @@ -705,34 +682,6 @@ module Gitlab end end - def aggregated_metrics - @aggregated_metrics ||= ::Gitlab::Usage::Metrics::Aggregates::Aggregate.new(recorded_at) - end - - def event_monthly_active_users(date_range) - data = { - action_monthly_active_users_project_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::PUSH_ACTION, - action_monthly_active_users_design_management: Gitlab::UsageDataCounters::TrackUniqueEvents::DESIGN_ACTION, - action_monthly_active_users_wiki_repo: Gitlab::UsageDataCounters::TrackUniqueEvents::WIKI_ACTION, - action_monthly_active_users_git_write: Gitlab::UsageDataCounters::TrackUniqueEvents::GIT_WRITE_ACTION - } - - data.each do |key, event| - data[key] = redis_usage_data { Gitlab::UsageDataCounters::TrackUniqueEvents.count_unique_events(event_action: event, **date_range) } - end - end - - def ide_monthly_active_users(date_range) - counter = Gitlab::UsageDataCounters::EditorUniqueCounter - - { - action_monthly_active_users_web_ide_edit: redis_usage_data { counter.count_web_ide_edit_actions(**date_range) }, - action_monthly_active_users_sfe_edit: redis_usage_data { counter.count_sfe_edit_actions(**date_range) }, - action_monthly_active_users_snippet_editor_edit: redis_usage_data { counter.count_snippet_editor_edit_actions(**date_range) }, - action_monthly_active_users_ide_edit: redis_usage_data { counter.count_edit_using_editor(**date_range) } - } - end - def distinct_count_service_desk_enabled_projects(time_period) project_creator_id_start = minimum_id(User) project_creator_id_finish = maximum_id(User) @@ -758,37 +707,6 @@ module Gitlab end # rubocop: enable CodeReuse/ActiveRecord - # rubocop: disable CodeReuse/ActiveRecord - def email_campaign_counts - # rubocop:disable UsageData/LargeTable - sent_emails = count(Users::InProductMarketingEmail.group(:track, :series)) - clicked_emails = count(Users::InProductMarketingEmail.where.not(cta_clicked_at: nil).group(:track, :series)) - - Users::InProductMarketingEmail::ACTIVE_TRACKS.keys.each_with_object({}) do |track, result| - series_amount = Namespaces::InProductMarketingEmailsService.email_count_for_track(track) - # rubocop: enable UsageData/LargeTable: - - 0.upto(series_amount - 1).map do |series| - sent_count = sent_in_product_marketing_email_count(sent_emails, track, series) - clicked_count = clicked_in_product_marketing_email_count(clicked_emails, track, series) - - result["in_product_marketing_email_#{track}_#{series}_sent"] = sent_count - result["in_product_marketing_email_#{track}_#{series}_cta_clicked"] = clicked_count unless track == 'experience' - end - end - end - # rubocop: enable CodeReuse/ActiveRecord - - def sent_in_product_marketing_email_count(sent_emails, track, series) - # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. - sent_emails.is_a?(Hash) ? sent_emails.fetch([track, series], 0) : sent_emails - end - - def clicked_in_product_marketing_email_count(clicked_emails, track, series) - # When there is an error with the query and it's not the Hash we expect, we return what we got from `count`. - clicked_emails.is_a?(Hash) ? clicked_emails.fetch([track, series], 0) : clicked_emails - end - def total_alert_issues # Remove prometheus table queries once they are deprecated # To be removed with https://gitlab.com/gitlab-org/gitlab/-/issues/217407. diff --git a/lib/gitlab/usage_data_counters.rb b/lib/gitlab/usage_data_counters.rb index 37c6e1af7c0..c2961de0eb9 100644 --- a/lib/gitlab/usage_data_counters.rb +++ b/lib/gitlab/usage_data_counters.rb @@ -2,9 +2,7 @@ module Gitlab module UsageDataCounters - COUNTERS = [].freeze - - COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES = [ + COUNTERS = [ PackageEventCounter, MergeRequestCounter, DesignsCounter, @@ -26,12 +24,8 @@ module Gitlab UnknownEvent = Class.new(UsageDataCounterError) class << self - def unmigrated_counters - self::COUNTERS - end - def counters - unmigrated_counters + migrated_counters + COUNTERS end def count(event_name) @@ -43,12 +37,6 @@ module Gitlab raise UnknownEvent, "Cannot find counter for event #{event_name}" end - - private - - def migrated_counters - COUNTERS_MIGRATED_TO_INSTRUMENTATION_CLASSES - end end end end diff --git a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb index 1e8918c7c96..eb040e9e819 100644 --- a/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/ci_template_unique_counter.rb @@ -10,13 +10,17 @@ module Gitlab::UsageDataCounters expanded_template_name = expand_template_name(template) return unless expanded_template_name - Gitlab::UsageDataCounters::HLLRedisCounter.track_event( - ci_template_event_name(expanded_template_name, config_source), values: project.id - ) + event_name = ci_template_event_name(expanded_template_name, config_source) + Gitlab::UsageDataCounters::HLLRedisCounter.track_event(event_name, values: project.id) namespace = project.namespace if Feature.enabled?(:route_hll_to_snowplow, namespace) - Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, user: user, project: project) + context = Gitlab::Tracking::ServicePingContext.new(data_source: :redis_hll, + event: event_name).to_context + label = 'redis_hll_counters.ci_templates.ci_templates_total_unique_counts_monthly' + Gitlab::Tracking.event(name, 'ci_templates_unique', namespace: namespace, + project: project, context: [context], user: user, + label: label) end end diff --git a/lib/gitlab/usage_data_counters/counter_events/package_events.yml b/lib/gitlab/usage_data_counters/counter_events/package_events.yml index a64d0ff7e24..f7ddc53f50d 100644 --- a/lib/gitlab/usage_data_counters/counter_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/counter_events/package_events.yml @@ -55,3 +55,5 @@ - i_package_terraform_module_delete_package - i_package_terraform_module_pull_package - i_package_terraform_module_push_package +- i_package_rpm_push_package +- i_package_rpm_pull_package diff --git a/lib/gitlab/usage_data_counters/known_events/common.yml b/lib/gitlab/usage_data_counters/known_events/common.yml index c13c7657576..c1720b26a22 100644 --- a/lib/gitlab/usage_data_counters/known_events/common.yml +++ b/lib/gitlab/usage_data_counters/known_events/common.yml @@ -127,6 +127,11 @@ category: testing redis_slot: testing aggregation: weekly +- name: i_testing_coverage_report_uploaded + category: testing + redis_slot: testing + aggregation: weekly + feature_flag: usage_data_ci_i_testing_coverage_report_uploaded # Project Management group - name: g_project_management_issue_title_changed category: issues_edit diff --git a/lib/gitlab/usage_data_counters/known_events/package_events.yml b/lib/gitlab/usage_data_counters/known_events/package_events.yml index debdbd8614f..ef8d02fa365 100644 --- a/lib/gitlab/usage_data_counters/known_events/package_events.yml +++ b/lib/gitlab/usage_data_counters/known_events/package_events.yml @@ -79,3 +79,11 @@ category: user_packages aggregation: weekly redis_slot: package +- name: i_package_rpm_user + category: user_packages + aggregation: weekly + redis_slot: package +- name: i_package_rpm_deploy_token + category: deploy_token_packages + aggregation: weekly + redis_slot: package diff --git a/lib/gitlab/usage_data_counters/known_events/work_items.yml b/lib/gitlab/usage_data_counters/known_events/work_items.yml index ee828fc0f72..d088b6d7e5a 100644 --- a/lib/gitlab/usage_data_counters/known_events/work_items.yml +++ b/lib/gitlab/usage_data_counters/known_events/work_items.yml @@ -19,6 +19,11 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_work_item_milestone + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity - name: users_updating_work_item_iteration # The event tracks an EE feature. # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. @@ -27,3 +32,11 @@ redis_slot: users aggregation: weekly feature_flag: track_work_items_activity +- name: users_updating_weight_estimate + # The event tracks an EE feature. + # It's added here so it can be aggregated into the CE/EE 'OR' aggregate metrics. + # It will report 0 for CE instances and should not be used with 'AND' aggregators. + category: work_items + redis_slot: users + aggregation: weekly + feature_flag: track_work_items_activity diff --git a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb index 8b9ca0fc220..d6e05f30a0d 100644 --- a/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb +++ b/lib/gitlab/usage_data_counters/kubernetes_agent_counter.rb @@ -8,6 +8,8 @@ module Gitlab class << self def increment_event_counts(events) + return unless events.present? + validate!(events) events.each do |event, incr| diff --git a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb index a0fd04596fc..b99c9ebb24f 100644 --- a/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb +++ b/lib/gitlab/usage_data_counters/work_item_activity_unique_counter.rb @@ -7,6 +7,7 @@ module Gitlab WORK_ITEM_TITLE_CHANGED = 'users_updating_work_item_title' WORK_ITEM_DATE_CHANGED = 'users_updating_work_item_dates' WORK_ITEM_LABELS_CHANGED = 'users_updating_work_item_labels' + WORK_ITEM_MILESTONE_CHANGED = 'users_updating_work_item_milestone' class << self def track_work_item_created_action(author:) @@ -25,6 +26,10 @@ module Gitlab track_unique_action(WORK_ITEM_LABELS_CHANGED, author) end + def track_work_item_milestone_changed_action(author:) + track_unique_action(WORK_ITEM_MILESTONE_CHANGED, author) + end + private def track_unique_action(action, author) diff --git a/lib/gitlab/utils.rb b/lib/gitlab/utils.rb index a67a0758257..d3055569ece 100644 --- a/lib/gitlab/utils.rb +++ b/lib/gitlab/utils.rb @@ -14,7 +14,10 @@ module Gitlab # Also see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/24223#note_284122580 # It also checks for ALT_SEPARATOR aka '\' (forward slash) def check_path_traversal!(path) - return unless path.is_a?(String) + return unless path + + path = path.to_s if path.is_a?(Gitlab::HashedPath) + raise PathTraversalAttackError, 'Invalid path' unless path.is_a?(String) path = decode_path(path) path_regex = %r{(\A(\.{1,2})\z|\A\.\.[/\\]|[/\\]\.\.\z|[/\\]\.\.[/\\]|\n)} @@ -164,9 +167,10 @@ module Gitlab end def deep_indifferent_access(data) - if data.is_a?(Array) + case data + when Array data.map(&method(:deep_indifferent_access)) - elsif data.is_a?(Hash) + when Hash data.with_indifferent_access else data @@ -174,9 +178,10 @@ module Gitlab end def deep_symbolized_access(data) - if data.is_a?(Array) + case data + when Array data.map(&method(:deep_symbolized_access)) - elsif data.is_a?(Hash) + when Hash data.deep_symbolize_keys else data diff --git a/lib/gitlab/utils/measuring.rb b/lib/gitlab/utils/measuring.rb index dc43d977a62..cfa09804b98 100644 --- a/lib/gitlab/utils/measuring.rb +++ b/lib/gitlab/utils/measuring.rb @@ -31,7 +31,7 @@ module Gitlab gc_stats: gc_stats, time_to_finish: time_to_finish, number_of_sql_calls: sql_calls_count, - memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss.to_f / 1024 / 1024} MiB", + memory_usage: "#{Gitlab::Metrics::System.memory_usage_rss[:total].to_f / 1024 / 1024} MiB", label: ::Prometheus::PidProvider.worker_id ) diff --git a/lib/gitlab/utils/strong_memoize.rb b/lib/gitlab/utils/strong_memoize.rb index 50b8428113d..6456ad08924 100644 --- a/lib/gitlab/utils/strong_memoize.rb +++ b/lib/gitlab/utils/strong_memoize.rb @@ -30,10 +30,10 @@ module Gitlab # end # strong_memoize_attr :trigger_from_token # - # strong_memoize_attr :enabled?, :enabled # def enabled? # Feature.enabled?(:some_feature) # end + # strong_memoize_attr :enabled?, :enabled # def strong_memoize(name) key = ivar(name) @@ -45,6 +45,16 @@ module Gitlab end end + def strong_memoize_with(name, *args) + container = strong_memoize(name) { {} } + + if container.key?(args) + container[args] + else + container[args] = yield + end + end + def strong_memoized?(name) instance_variable_defined?(ivar(name)) end @@ -58,23 +68,8 @@ module Gitlab def strong_memoize_attr(method_name, member_name = nil) member_name ||= method_name - if method_defined?(method_name) || private_method_defined?(method_name) - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) - else - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :queue_strong_memoize, self, method_name, member_name) - end - end - - def method_added(method_name) - super - - if member_name = StrongMemoize - .send(:strong_memoize_queue, self).delete(method_name) # rubocop:disable GitlabSecurity/PublicSend - StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend - :do_strong_memoize, self, method_name, member_name) - end + StrongMemoize.send( # rubocop:disable GitlabSecurity/PublicSend + :do_strong_memoize, self, method_name, member_name) end end @@ -88,9 +83,10 @@ module Gitlab # # Depending on a type ensure that there's a single memory allocation def ivar(name) - if name.is_a?(Symbol) + case name + when Symbol name.to_s.prepend("@").to_sym - elsif name.is_a?(String) + when String :"@#{name}" else raise ArgumentError, "Invalid type of '#{name}'" @@ -100,14 +96,6 @@ module Gitlab class <<self private - def strong_memoize_queue(klass) - klass.instance_variable_get(:@strong_memoize_queue) || klass.instance_variable_set(:@strong_memoize_queue, {}) - end - - def queue_strong_memoize(klass, method_name, member_name) - strong_memoize_queue(klass)[method_name] = member_name - end - def do_strong_memoize(klass, method_name, member_name) method = klass.instance_method(method_name) diff --git a/lib/gitlab/web_hooks/recursion_detection.rb b/lib/gitlab/web_hooks/recursion_detection.rb index 031d9ec6ec4..7e79283757f 100644 --- a/lib/gitlab/web_hooks/recursion_detection.rb +++ b/lib/gitlab/web_hooks/recursion_detection.rb @@ -41,7 +41,7 @@ module Gitlab ::Gitlab::Redis::SharedState.with do |redis| redis.multi do |multi| - multi.sadd(cache_key, hook.id) + multi.sadd?(cache_key, hook.id) multi.expire(cache_key, TOUCH_CACHE_TTL) end end diff --git a/lib/gitlab/workhorse.rb b/lib/gitlab/workhorse.rb index 906439d5e71..0d5daeefe90 100644 --- a/lib/gitlab/workhorse.rb +++ b/lib/gitlab/workhorse.rb @@ -33,7 +33,12 @@ module Gitlab GitalyServer: { address: Gitlab::GitalyClient.address(repository.storage), token: Gitlab::GitalyClient.token(repository.storage), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor(user), + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } } @@ -252,7 +257,12 @@ module Gitlab { address: Gitlab::GitalyClient.address(repository.shard), token: Gitlab::GitalyClient.token(repository.shard), - features: Feature::Gitaly.server_feature_flags(repository.project) + features: Feature::Gitaly.server_feature_flags( + user: ::Feature::Gitaly.user_actor, + repository: repository, + project: ::Feature::Gitaly.project_actor(repository.container), + group: ::Feature::Gitaly.group_actor(repository.container) + ) } end |